BucketCache.h

Go to the documentation of this file.

// # BucketCache.h: Cache for buckets in a part of a file
// # Copyright (C) 1994,1995,1996,1999,2000,2001
// # Associated Universities, Inc. Washington DC, USA.
// #
// # This library is free software; you can redistribute it and/or modify it
// # under the terms of the GNU Library General Public License as published by
// # the Free Software Foundation; either version 2 of the License, or (at your
// # option) any later version.
// #
// # This library is distributed in the hope that it will be useful, but WITHOUT
// # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
// # License for more details.
// #
// # You should have received a copy of the GNU Library General Public License
// # along with this library; if not, write to the Free Software Foundation,
// # Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
// #
// # Correspondence concerning AIPS++ should be addressed as follows:
// #        Internet email: casa-feedback@nrao.edu.
// #        Postal address: AIPS++ Project Office
// #                        National Radio Astronomy Observatory
// #                        520 Edgemont Road
// #                        Charlottesville, VA 22903-2475 USA
 
#ifndef CASA_BUCKETCACHE_H
#define CASA_BUCKETCACHE_H
 
// # Includes
#include <casacore/casa/aips.h>
#include <casacore/casa/IO/BucketFile.h>
#include <casacore/casa/Containers/Block.h>
#include <casacore/casa/OS/CanonicalConversion.h>
 
// # Forward clarations
#include <casacore/casa/iosfwd.h>
 
namespace casacore {  // # NAMESPACE CASACORE - BEGIN
 
// <summary>
// Define the type of the static read and write function.
// </summary>
// <use visibility=export>
// <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="" demos="">
// </reviewed>
 
// <synopsis>
// The BucketCache class needs a way to convert its data from local
// to canonical format and vice-versa. This is done by callback
// functions defined at construction time.
// <p>
// The ToLocal callback function has to allocate a buffer of the correct
// size and to copy/convert the canonical data in the input buffer to
// this buffer. The pointer this newly allocated buffer has to be returned.
// The BucketCache class keeps this pointer in the cache block.
// <p>
// The FromLocal callback function has to copy/convert the data from the
// buffer in local format to the buffer in canonical format. It should
// NOT delete the buffer; that has to be done by the DeleteBuffer function.
// <p>
// The AddBuffer callback function has to create (and initialize) a
// buffer to be added to the file and cache.
// When the file gets extended, BucketCache only registers the new size,
// but does not werite anything. When a bucket is read between the
// actual file size and the new file size, the AddBuffer callback function
// is called to create a buffer and possibly initialize it.
// <p>
// The DeleteBuffer callback function has to delete the buffer
// allocated by the ToLocal function.
// <p>
// The functions get a pointer to the owner object, which was provided
// at construction time. The callback function has to cast this to the
// correct type and can use it thereafter.
// <br>
// C++ supports pointers to members, but it is a bit hard. Therefore pointers
// to static members are used (which are simple pointers to functions).
// A pointer to the owner object is also passed to let the static function
// call the correct member function (when needed).
// </synopsis>
//
// <example>
// See class <linkto class=BucketCache>BucketCache</linkto>.
// </example>
 
// <group name=BucketCache_CallBack>
typedef char* (*BucketCacheToLocal)(void* ownerObject, const char* canonical);
typedef void (*BucketCacheFromLocal)(void* ownerObject, char* canonical, const char* local);
typedef char* (*BucketCacheAddBuffer)(void* ownerObject);
typedef void (*BucketCacheDeleteBuffer)(void* ownerObject, char* buffer);
// </group>
 
// <summary>
// Cache for buckets in a part of a file
// </summary>
 
// <use visibility=export>
 
// <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="" demos="">
// </reviewed>
 
// <prerequisite>
// # Classes you should understand before using this one.
//   <li> <linkto class=BucketFile>BucketFile</linkto>
// </prerequisite>
 
// <etymology>
// BucketCache implements a cache for buckets in (a part of) a file.
// </etymology>
 
// <synopsis>
// A cache may allow more efficient quasi-random IO.
// It can, for instance, be used when a limited number of blocks
// in a file have to be accessed again and again.
// <p>
// The class BucketCache provides such a cache. It can be used on a
// consecutive part of a file as long as that part is not simultaneously
// accessed in another way (including another BucketCache object).
// <p>
// BucketCache stores the data as given.
// It uses <linkto group=BucketCache_CallBack>callback functions</linkto>
// to allocate/delete buffers and to convert the data to/from local format.
// <p>
// When a new bucket is needed and all slots in the cache are used,
// BucketCache will remove the least recently used bucket from the
// cache. When the dirty flag is set, it will first be written.
// <p>
// BucketCache maintains a list of free buckets. Initially this list is
// empty. When a bucket is removed, it is added to the free list.
// AddBucket will take buckets from the free list before extending the file.
// <p>
// Since it is possible to handle only a part of a file by a BucketCache
// object, it is also possible to have multiple BucketCache objects on
// the same file (as long as they access disjoint parts of the file).
// Each BucketCache object can have its own bucket size. This can,
// for example, be used to have tiled arrays with different tile shapes
// in the same file.
// <p>
// Statistics are kept to know how efficient the cache is working.
// It is possible to initialize and show the statistics.
// </synopsis>
 
// <motivation>
// A cache may reduce IO traffix considerably.
// Furthermore it is more efficient to keep a cache in local format.
// In that way conversion to/from local only have to be done when
// data gets read/written. It also allows for precalculations.
// </motivation>
 
// <example>
// <srcblock>
//  // Define the callback function for reading a bucket.
//  char* bToLocal (void*, const char* data)
//  {
//    char* ptr = new char[32768];
//    memcpy (ptr, data, 32768);
//    return ptr;
//  }
//  // Define the callback function for writing a bucket.
//  void bFromLocal (void*, char* data, const char* local)
//  {
//    memcpy (data, local, 32768);
//  }
//  // Define the callback function for initializing a new bucket.
//  char* bAddBuffer (void*)
//  {
//    char* ptr = new char[32768];
//    for (uInt i=0; i++; i<32768) {
//  ptr[i] = 0;
//    }
//    return ptr;
//  }
//  // Define the callback function for deleting a bucket.
//  void bDeleteBuffer (void*, char* buffer)
//  {
//    delete [] buffer;
//  }
//
//  void someFunc()
//  {
//    // Open the filebuf.
//    BucketFile file(...);
//    file.open();
//    uInt i;
//    // Create a cache for the part of the file starting at offset 512
//    // consisting of 1000 buckets. The cache consists of 10 buckets.
//    // Each bucket is 32768 bytes.
//    BucketCache cache (&file, 512, 32768, 1000, 10, 0,
//                       bToLocal, bFromLocal, bAddBuffer, bDeleteBuffer);
//    // Write all buckets into the file.
//    for (i=0; i<100; i++) {
//      char* buf = new char[32768];
//      cache.addBucket (buf);
//    }
//    Flush the cache to write all buckets in it.
//    cache.flush();
//    // Read all buckets from the file.
//    for (i=0; i<1000; i++) {
//      char* buf = cache.getBucket(i);
//      ...
//    }
//    cout << cache.nBucket() << endl;
//  }
// </srcblock>
// </example>
 
// <todo asof="$DATE:$">
//   <li> When ready, use HashMap for the internal maps.
// </todo>
 
class BucketCache {
 public:
  // Create the cache for (a part of) a file.
  // The file part used starts at startOffset. Its length is
  // bucketSize*nrOfBuckets bytes.
  // When the file is smaller, the remainder is indicated as an extension
  // similarly to the behaviour of function extend.
  BucketCache(BucketFile* file, Int64 startOffset, uInt bucketSize, uInt nrOfBuckets,
              uInt cacheSize, void* ownerObject, BucketCacheToLocal readCallBack,
              BucketCacheFromLocal writeCallBack, BucketCacheAddBuffer addCallBack,
              BucketCacheDeleteBuffer deleteCallBack);
 
  ~BucketCache();
 
  // Flush the cache from the given slot on.
  // By default the entire cache is flushed.
  // When the entire cache is flushed, possible remaining uninitialized
  // buckets will be initialized first.
  // A True status is returned when buckets had to be written.
  Bool flush(uInt fromSlot = 0);
 
  // Clear the cache from the given slot on.
  // By default the entire cache is cleared.
  // It will remove the buckets in the cleared part.
  // If wanted and needed, the buckets are flushed to the file
  // before removing them.
  // It can be used to enforce rereading buckets from the file.
  void clear(uInt fromSlot = 0, Bool doFlush = True);
 
  // Resize the cache.
  // When the cache gets smaller, the latter buckets are cached out.
  // It does not take "least recently used" into account.
  void resize(uInt cacheSize);
 
  // Resynchronize the object (after another process updated the file).
  // It clears the cache (so all data will be reread) and sets
  // the new sizes.
  void resync(uInt nrBucket, uInt nrOfFreeBucket, Int firstFreeBucket);
 
  // Get the current nr of buckets in the file.
  uInt nBucket() const;
 
  // Get the current cache size (in buckets).
  uInt cacheSize() const;
 
  // Set the dirty bit for the current bucket.
  void setDirty();
 
  // Make another bucket current.
  // When no more cache slots are available, the one least recently
  // used is flushed.
  // The data in the bucket is converted using the ToLocal callback
  // function. When the bucket does not exist yet in the file, it
  // gets added and initialized using the AddBuffer callback function.
  // A pointer to the data in converted format is returned.
  char* getBucket(uInt bucketNr);
 
  // Extend the file with the given number of buckets.
  // The buckets get initialized when they are acquired
  // (using getBucket) for the first time.
  void extend(uInt nrBucket);
 
  // Add a bucket to the file and make it the current one.
  // When no more cache slots are available, the one least recently
  // used is flushed.
  // <br> When no free buckets are available, the file will be
  // extended with one bucket. It returns the new bucket number.
  // The buffer must have been allocated on the heap.
  // It will get part of the cache; its contents are not copied.
  // Thus the buffer should hereafter NOT be used for other purposes.
  // It will be deleted later via the DeleteBuffer callback function.
  // The data is copied into the bucket. A pointer to the data in
  // local format is returned.
  uInt addBucket(char* data);
 
  // Remove the current bucket; i.e. add it to the beginning of the
  // free bucket list.
  void removeBucket();
 
  // Get a part from the file outside the cached area.
  // It is checked if that part is indeed outside the cached file area.
  void get(char* buf, uInt length, Int64 offset);
 
  // Put a part from the file outside the cached area.
  // It is checked if that part is indeed outside the cached file area.
  void put(const char* buf, uInt length, Int64 offset);
 
  // Get the bucket number of the first free bucket.
  // -1 = no free buckets.
  Int firstFreeBucket() const;
 
  // Get the number of free buckets.
  uInt nFreeBucket() const;
 
  // (Re)initialize the cache statistics.
  void initStatistics();
 
  // Show the statistics.
  void showStatistics(ostream& os) const;
 
 private:
  // The file used.
  BucketFile* its_file;
  // The owner object.
  void* its_Owner;
  // The read callback function.
  BucketCacheToLocal its_ReadCallBack;
  // The write callback function.
  BucketCacheFromLocal its_WriteCallBack;
  // The add bucket callback function.
  BucketCacheAddBuffer its_InitCallBack;
  // The delete callback function.
  BucketCacheDeleteBuffer its_DeleteCallBack;
  // The starting offsets of the buckets in the file.
  Int64 its_StartOffset;
  // The bucket size.
  uInt its_BucketSize;
  // The current nr of buckets in the file.
  uInt its_CurNrOfBuckets;
  // The new nr of buckets in the file (after extension).
  uInt its_NewNrOfBuckets;
  // The size of the cache (i.e. #buckets fitting in it).
  uInt its_CacheSize;
  // The nr of slots used in the cache.
  uInt its_CacheSizeUsed;
  // The cache itself.
  Block<char*> its_Cache;
  // The cache slot actually used.
  uInt its_ActualSlot;
  // The slot numbers of the buckets in the cache (-1 = not in cache).
  Block<Int> its_SlotNr;
  // The buckets in the cache.
  Block<uInt> its_BucketNr;
  // Determine if a block is dirty (i.e. changed) (1=dirty).
  Block<uInt> its_Dirty;
  // Determine when a block is used for the last time.
  Block<uInt> its_LRU;
  // The Least Recently Used counter.
  uInt its_LRUCounter;
  // The internal buffer.
  char* its_Buffer;
  // The number of free buckets.
  uInt its_NrOfFree;
  // The first free bucket (-1 = no free buckets).
  Int its_FirstFree;
  // The statistics.
  uInt naccess_p;
  uInt nread_p;
  uInt ninit_p;
  uInt nwrite_p;
 
  // Copy constructor is not possible.
  BucketCache(const BucketCache&);
 
  // Assignment is not possible.
  BucketCache& operator=(const BucketCache&);
 
  // Set the LRU information for the current slot.
  void setLRU();
 
  // Get a cache slot for the bucket.
  void getSlot(uInt bucketNr);
 
  // Write a bucket.
  void writeBucket(uInt slotNr);
 
  // Read a bucket.
  void readBucket(uInt slotNr);
 
  // Initialize the bucket buffer.
  // The uninitialized buckets before this bucket are also initialized.
  // It returns a pointer to the buffer.
  void initializeBuckets(uInt bucketNr);
 
  // Check if the offset of a non-cached part is correct.
  void checkOffset(uInt length, Int64 offset) const;
};
 
inline uInt BucketCache::cacheSize() const { return its_CacheSize; }
 
inline Int BucketCache::firstFreeBucket() const { return its_FirstFree; }
 
inline uInt BucketCache::nFreeBucket() const { return its_NrOfFree; }
 
}  // namespace casacore
 
#endif