BaseTable.h

Go to the documentation of this file.

// # BaseTable.h: Abstract base class for tables
// # Copyright (C) 1994,1995,1996,1997,1998,2000,2001,2002,2003
// # Associated Universities, Inc. Washington DC, USA.
// #
// # This library is free software; you can redistribute it and/or modify it
// # under the terms of the GNU Library General Public License as published by
// # the Free Software Foundation; either version 2 of the License, or (at your
// # option) any later version.
// #
// # This library is distributed in the hope that it will be useful, but WITHOUT
// # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
// # License for more details.
// #
// # You should have received a copy of the GNU Library General Public License
// # along with this library; if not, write to the Free Software Foundation,
// # Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
// #
// # Correspondence concerning AIPS++ should be addressed as follows:
// #        Internet email: casa-feedback@nrao.edu.
// #        Postal address: AIPS++ Project Office
// #                        National Radio Astronomy Observatory
// #                        520 Edgemont Road
// #                        Charlottesville, VA 22903-2475 USA
 
#ifndef TABLES_BASETABLE_H
#define TABLES_BASETABLE_H
 
// # Includes
#include <casacore/casa/aips.h>
#include <casacore/tables/Tables/TableInfo.h>
#include <casacore/tables/Tables/TableDesc.h>
#include <casacore/tables/Tables/StorageOption.h>
#include <casacore/casa/Utilities/Compare.h>
#include <casacore/casa/BasicSL/String.h>
#include <casacore/casa/IO/FileLocker.h>
#include <casacore/casa/Arrays/ArrayFwd.h>
#include <memory>
 
#ifdef HAVE_MPI
#include <mpi.h>
#endif
 
namespace casacore {  // # NAMESPACE CASACORE - BEGIN
 
// # Forward Declarations
class RefTable;
// class TableDesc;  !Forward declaration not recognized SGI compiler
class TableLock;
class BaseColumn;
class ColumnDesc;
class TableRecord;
class Record;
class TableExprNode;
class BaseTableIterator;
class DataManager;
class IPosition;
template <class T>
class Block;
class AipsIO;
 
// <summary>
// Abstract base class for tables
// </summary>
 
// <use visibility=local>
 
// <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
// </reviewed>
 
// <prerequisite>
// # Classes you should understand before using this one.
//   <li> Table
//   <li> Sort
//   <li> TableExprNode
// </prerequisite>
 
// <etymology>
// BaseTable is the (abstract) base class for different kind of tables.
// </etymology>
 
// <synopsis>
// BaseTables defines many virtual functions, which are actually
// implemented in the underlying table classes like PlainTable and
// RefTable. Other functions like sort and select are implemented
// in BaseTable itself.
//
// The functions in BaseTable and its derived classes can only be
// used by the table system classes. All user access is via the
// envelope class Table, which references (counted) BaseTable.
// </synopsis>
 
// <todo asof="$DATE:$">
// # A List of bugs, limitations, extensions or planned refinements.
//   <li> Implement function renameColumn, removeColumn.
// </todo>
 
class BaseTable : public std::enable_shared_from_this<BaseTable> {
 public:
  // Initialize the object.
  BaseTable(const String& tableName, int tableOption, rownr_t nrrow);
 
#ifdef HAVE_MPI
  // MPI version of the constructor
  BaseTable(MPI_Comm mpiComm, const String& tableName, int tableOption, rownr_t nrrow);
#endif
 
  // Common code shared by the MPI constructor and non-MPI constructor
  void BaseTableCommon(const String& tableName, int tableOption, rownr_t nrrow);
 
  // The destructor will delete the table if needed.
  virtual ~BaseTable();
 
  // Copy constructor is forbidden, because copying a table requires
  // some more knowledge (like table name of result).
  BaseTable(const BaseTable&) = delete;
 
  // Assignment is forbidden, because copying a table requires
  // some more knowledge (like table name of result).
  BaseTable& operator=(const BaseTable&) = delete;
 
  // Is the table a null table?
  // By default it is not.
  virtual Bool isNull() const;
 
  // Reopen the table for read/write.
  virtual void reopenRW() = 0;
 
  // Indicate we will only modify data values in existing tiled columns
  virtual void changeTiledDataOnly();
 
  // Is the table stored in big or little endian format?
  virtual Bool asBigEndian() const = 0;
 
  // Get the storage option used for the table.
  virtual const StorageOption& storageOption() const = 0;
 
  // Is the table in use (i.e. open) in another process?
  // If <src>checkSubTables</src> is set, it is also checked if
  // a subtable is used in another process.
  virtual Bool isMultiUsed(Bool checkSubTables) const = 0;
 
  // Get the locking info.
  virtual const TableLock& lockOptions() const = 0;
 
  // Merge the given lock info with the existing one.
  virtual void mergeLock(const TableLock& lockOptions) = 0;
 
  // Has this process the read or write lock, thus can the table
  // be read or written safely?
  virtual Bool hasLock(FileLocker::LockType) const = 0;
 
  // Try to lock the table for read or write access.
  virtual Bool lock(FileLocker::LockType, uInt nattempts) = 0;
 
  // Unlock the table. This will also synchronize the table data,
  // thus force the data to be written to disk.
  virtual void unlock() = 0;
 
  // Flush the table, i.e. write it to disk.
  virtual void flush(Bool fsync, Bool recursive) = 0;
 
  // Resync the Table object with the table file.
  virtual void resync() = 0;
 
  // Get the modify counter.
  virtual uInt getModifyCounter() const = 0;
 
  // Set the table to being changed. By default it does nothing.
  virtual void setTableChanged();
 
  // Do not write the table (used in in case of exceptions).
  void doNotWrite() { noWrite_p = True; }
 
  // Test if this table is writable.
  // This tells if values can be put into a column.
  virtual Bool isWritable() const = 0;
 
  // Test if the given column is writable.
  // <group>
  Bool isColumnWritable(const String& columnName) const;
  Bool isColumnWritable(uInt columnIndex) const;
  // </group>
 
  // Test if the given column is stored (otherwise it is virtual).
  // <group>
  Bool isColumnStored(const String& columnName) const;
  Bool isColumnStored(uInt columnIndex) const;
  // </group>
 
  // Get the table name.
  const String& tableName() const { return name_p; }
 
  // Get the names of the tables this table consists of.
  // The default implementation adds the name of this table to the block.
  virtual void getPartNames(Block<String>& names, Bool recursive) const;
 
  // Rename the table.
  // The following options can be given:
  // <dl>
  // <dt> Table::Update
  // <dd> A table with this name must already exists, which will be
  //      overwritten. When succesfully renamed, the table is unmarked
  //      for delete (if necessary).
  // <dt> Table::New
  // <dd> When a table with this name exists, it will be overwritten.
  //      When succesfully renamed, the table is unmarked
  //      for delete (if necessary).
  // <dt> Table::NewNoReplace
  // <dd> When a table with this name already exists, an exception
  //      is thrown. When succesfully renamed, the table
  //      is unmarked for delete (if necessary).
  // <dt> Table::Scratch
  // <dd> Same as Table::New, but followed by markForDelete().
  // </dl>
  // The rename function in this base class renames the table file.
  // In a derived class (e.g. PlainTable) the function should also
  // be implemented to rename subtables in its keywords.
  virtual void rename(const String& newName, int tableOption);
 
  // Copy the table and all its subtables.
  // The default implementation of deepCopy is to call copy.
  // The following options can be given:
  // <dl>
  // <dt> Table::New
  // <dd> When a table with this name exists, it will be overwritten.
  // <dt> Table::NewNoReplace
  // <dd> When a table with this name already exists, an exception
  //      is thrown.
  // <dt> Table::Scratch
  // <dd> Same as Table::New, but followed by markForDelete().
  // </dl>
  // <group>
  virtual void copy(const String& newName, int tableOption) const;
  virtual void deepCopy(const String& newName, const Record& dataManagerInfo, const StorageOption&,
                        int tableOption, Bool valueCopy, int endianFormat, Bool noRows) const;
  // </group>
 
  // Get the table type.
  // By default it returns Table::Plain.
  virtual int tableType() const;
 
  // Get the table option.
  int tableOption() const { return option_p; }
 
  // Mark the table for delete.
  // This means that the underlying table gets deleted when it is
  // actually destructed.
  // The scratchCallback function is called when needed.
  void markForDelete(Bool callback, const String& oldName);
 
  // Unmark the table for delete.
  // This means the underlying table does not get deleted when destructed.
  // The scratchCallback function is called when needed.
  void unmarkForDelete(Bool callback, const String& oldName);
 
  // Test if the table is marked for delete.
  Bool isMarkedForDelete() const { return delete_p; }
 
  // Get the table description.
  const TableDesc& tableDesc() const { return (!tdescPtr_p ? makeEmptyTableDesc() : *tdescPtr_p); }
 
  // Get the actual table description.
  virtual TableDesc actualTableDesc() const = 0;
 
  // Get the data manager info.
  virtual Record dataManagerInfo() const = 0;
 
  // Show the table structure (implementation of Table::showStructure).
  void showStructure(std::ostream&, Bool showDataMan, Bool showColumns, Bool showSubTables,
                     Bool sortColumns, Bool cOrder);
 
  // Get readonly access to the table keyword set.
  virtual TableRecord& keywordSet() = 0;
 
  // Get read/write access to the table keyword set.
  // This requires that the table is locked (or it gets locked
  // when using AutoLocking mode).
  virtual TableRecord& rwKeywordSet() = 0;
 
  // Get access to the TableInfo object.
  TableInfo& tableInfo() { return info_p; }
 
  // Get the table info of the table with the given name.
  // An empty object is returned when the table is unknown.
  static TableInfo tableInfo(const String& tableName);
 
  // Write the TableInfo object.
  virtual void flushTableInfo();
 
  // Get number of rows.
  rownr_t nrow() const { return nrrow_p; }
 
  // Get a column object using its index.
  virtual BaseColumn* getColumn(uInt columnIndex) const = 0;
 
  // Get a column object using its name.
  virtual BaseColumn* getColumn(const String& columnName) const = 0;
 
  // Test if it is possible to add a row to this table.
  virtual Bool canAddRow() const;
 
  // Add one or more rows and possibly initialize them.
  // This will fail for tables not supporting addition of rows.
  virtual void addRow(rownr_t nrrow = 1, Bool initialize = True);
 
  // Test if it is possible to remove a row from this table.
  virtual Bool canRemoveRow() const;
 
  // Remove rows.
  // This will fail for tables not supporting removal of rows.
  // <note role=tip>
  // The following code fragments do NOT have the same result:
  // <srcblock>
  //    tab.removeRow (10);      // remove row 10
  //    tab.removeRow (20);      // remove row 20, which was 21
  //
  //    Vector<rownr_t> vec(2);
  //    vec(0) = 10;
  //    vec(1) = 20;
  //    tab.removeRow (vec);     // remove row 10 and 20
  // </srcblock>
  // because in the first fragment removing row 10 turns the former
  // row 21 into row 20.
  // </note>
  // <group>
  virtual void removeRow(rownr_t rownr);
  void removeRow(const Vector<rownr_t>& rownrs);
  void removeRow(const Vector<uInt>& rownrs);
  // </group>
 
  // Find the data manager with the given name or for the given column.
  virtual DataManager* findDataManager(const String& name, Bool byColumn) const = 0;
 
  // Select rows using the given expression (which can be null).
  // Skip first <src>offset</src> matching rows.
  // Return at most <src>maxRow</src> matching rows.
  std::shared_ptr<BaseTable> select(const TableExprNode&, rownr_t maxRow, rownr_t offset);
 
  // Select maxRow rows and skip first offset rows. maxRow=0 means all.
  std::shared_ptr<BaseTable> select(rownr_t maxRow, rownr_t offset);
 
  // Select rows using a vector of row numbers.
  std::shared_ptr<BaseTable> select(const Vector<rownr_t>& rownrs);
 
  // Select rows using a mask block.
  // The length of the block must match the number of rows in the table.
  // If True, the corresponding row will be selected.
  std::shared_ptr<BaseTable> select(const Block<Bool>& mask);
 
  // Project the given columns (i.e. select the columns).
  std::shared_ptr<BaseTable> project(const Block<String>& columnNames);
 
  // Do logical operations on a table.
  // <group>
  // intersection with another table
  std::shared_ptr<BaseTable> tabAnd(BaseTable*);
  // union with another table
  std::shared_ptr<BaseTable> tabOr(BaseTable*);
  // subtract another table
  std::shared_ptr<BaseTable> tabSub(BaseTable*);
  // xor with another table
  std::shared_ptr<BaseTable> tabXor(BaseTable*);
  // take complement
  std::shared_ptr<BaseTable> tabNot();
  // </group>
 
  // Sort a table on one or more columns of scalars.
  std::shared_ptr<BaseTable> sort(const Block<String>& columnNames,
                                  const Block<std::shared_ptr<BaseCompare>>& compareObjects,
                                  const Block<Int>& sortOrder, int sortOption,
                                  std::shared_ptr<Vector<rownr_t>> sortIterBoundaries = nullptr,
                                  std::shared_ptr<Vector<size_t>> sortIterKeyIdxChange = nullptr);
 
  // Create an iterator.
  BaseTableIterator* makeIterator(const Block<String>& columnNames,
                                  const Block<std::shared_ptr<BaseCompare>>&,
                                  const Block<Int>& orders, int option,
                                  bool cacheIterationBoundaries = false);
 
  // Add one or more columns to the table.
  // The default implementation throws an "invalid operation" exception.
  // <group>
  virtual void addColumn(const ColumnDesc& columnDesc, Bool addToParent);
  virtual void addColumn(const ColumnDesc& columnDesc, const String& dataManager, Bool byName,
                         Bool addToParent);
  virtual void addColumn(const ColumnDesc& columnDesc, const DataManager& dataManager,
                         Bool addToParent);
  virtual void addColumn(const TableDesc& tableDesc, const DataManager& dataManager,
                         Bool addToParent);
  // </group>
 
  // Add one or more columns to the table.
  // The data manager to use is described in the record.
  void addColumns(const TableDesc& tableDesc, const Record& dmInfo, Bool addToParent);
 
  // Test if columns can be removed.
  virtual Bool canRemoveColumn(const Vector<String>& columnNames) const = 0;
 
  // Remove columns.
  virtual void removeColumn(const Vector<String>& columnNames) = 0;
 
  // Check if the set of columns can be removed.
  // It checks if columns have not been specified twice and it
  // checks if they exist.
  // If the flag is set an exception is thrown if errors are found.
  Bool checkRemoveColumn(const Vector<String>& columnNames, Bool throwException) const;
 
  // Test if a column can be renamed.
  virtual Bool canRenameColumn(const String& columnName) const = 0;
 
  // Rename a column.
  virtual void renameColumn(const String& newName, const String& oldName) = 0;
 
  // Rename a hypercolumn.
  virtual void renameHypercolumn(const String& newName, const String& oldName) = 0;
 
  // Get a vector of row numbers.
  // By default it returns the row numbers 0..nrrow()-1.
  // It needs to be implemented for RefTable only.
  virtual Vector<rownr_t> rowNumbers() const;
 
  // Get pointer to root table (i.e. parent of a RefTable).
  // Default it is this table.
  // It is meant for the reference tables after a select or sort which
  // can then still name their parent as the root.
  virtual BaseTable* root();
 
  // Tell if the table is in row order.
  // By default it is, since normally a table is always in row order.
  // It is meant for RefTable-s, where the rows can be in
  // another (sorted) order.
  virtual Bool rowOrder() const;
 
  // By the default the table cannot return the storage of rownrs.
  // That can only be done by a RefTable, where it is implemented.
  virtual Vector<rownr_t>& rowStorage();
 
  // Adjust the row numbers to be the actual row numbers in the
  // root table. This is, for instance, used when a RefTable is sorted.
  // Optionally it also determines if the resulting rows are in order.
  virtual Bool adjustRownrs(rownr_t nrrow, Vector<rownr_t>& rownrs, Bool determineOrder) const;
 
  // Do the actual sort.
  // The default implementation is suitable for almost all cases.
  // Only in RefTable a smarter implementation is provided.
  virtual std::shared_ptr<BaseTable> doSort(Block<BaseColumn*>&,
                                            const Block<std::shared_ptr<BaseCompare>>&,
                                            const Block<Int>& sortOrder, int sortOption,
                                            std::shared_ptr<Vector<rownr_t>> sortIterBoundaries,
                                            std::shared_ptr<Vector<size_t>> sortIterKeyIdxChange);
 
  // Create a RefTable object.
  std::shared_ptr<RefTable> makeRefTable(Bool rowOrder, rownr_t initialNrrow);
 
  // Check if the row number is valid.
  // It throws an exception if out of range.
  void checkRowNumber(rownr_t rownr) const {
    if (rownr >= nrrow_p + nrrowToAdd_p) checkRowNumberThrow(rownr);
  }
 
  // Get the table's trace-id.
  int traceId() const { return itsTraceId; }
 
 protected:
  std::weak_ptr<BaseTable> thisPtr_p;     // # pointer to itself (to make shared_ptr)
  rownr_t nrrow_p;                        // # #rows in this table
  rownr_t nrrowToAdd_p;                   // # #rows to be added
  std::shared_ptr<TableDesc> tdescPtr_p;  // # Pointer to table description
  String name_p;                          // # table name
  int option_p;                           // # Table constructor option
  Bool noWrite_p;                         // # False = do not write the table
  Bool delete_p;                          // # True = delete when destructed
  TableInfo info_p;                       // # Table information (type, etc.)
  Bool madeDir_p;                         // # True = table dir has been created
  int itsTraceId;                         // # table-id for TableTrace tracing
 
  // Do the callback for scratch tables (if callback is set).
  void scratchCallback(Bool isScratch, const String& oldName) const;
 
  // Create the table directory when needed (and possible).
  // When the file already exists, check if it is a directory.
  // It returns True when it actually created the directory.
  Bool makeTableDir();
 
  // Make a true deep copy of the table.
  // The table is flushed before making the copy.
  void trueDeepCopy(const String& newName, const Record& dataManagerInfo, const StorageOption&,
                    int tableOption, int endianFormat, Bool noRows) const;
 
  // Prepare for copying or renaming a table.
  // It checks if the target table already exists and removes it
  // when necessary.
  void prepareCopyRename(const String& newName, int tableOption) const;
 
  // Rename the subtables (used by rename function).
  virtual void renameSubTables(const String& newName, const String& oldName);
 
  // Check if the table already exists.
  // Throw an exception if so.
  void throwIfTableExists();
 
  // Test if the table is opened for write.
  Bool openedForWrite() const;
 
  // Start writing a table. It does a putstart and writes <src>nrrow_p</src>.
  // It should be ended by calling <src>writeEnd</src>.
  void writeStart(AipsIO&, Bool bigEndian);
 
  // End writing a table.
  void writeEnd(AipsIO&);
 
  // Should the table be written.
  // This flag is False if an exception was thrown.
  Bool shouldNotWrite() const { return noWrite_p; }
 
  // Read the TableInfo object.
  void getTableInfo();
 
 private:
  // Show a possible extra table structure header.
  // It is used by e.g. RefTable to show which table is referenced.
  virtual void showStructureExtra(std::ostream&) const;
 
  // Show the info of the given columns.
  // Sort the columns if needed.
  void showColumnInfo(ostream& os, const TableDesc&, uInt maxNameLength,
                      const Array<String>& columnNames, Bool sort, Bool cOrder) const;
 
  // Throw an exception for checkRowNumber.
  void checkRowNumberThrow(rownr_t rownr) const;
 
  // Check if the tables combined in a logical operation have the
  // same root.
  void logicCheck(BaseTable* that);
 
  // Get the rownrs of the table in ascending order to be
  // used in the logical operation on the table.
  Vector<rownr_t> logicRows();
 
  // Make an empty table description.
  // This is used if one asks for the description of a NullTable.
  // Creating an empty TableDesc in the NullTable takes too much time.
  // Furthermore it causes static initialization order problems.
  const TableDesc& makeEmptyTableDesc() const;
 
  // Make the name absolute.
  // It first checks if the name contains valid characters (not only . and /).
  String makeAbsoluteName(const String& name) const;
 
#ifdef HAVE_MPI
  // MPI communicator for parallel I/O.
  // When using an MPI-disabled casacore, MPI applications have always been
  // able to create Tables from each rank independently. Defaulting this
  // communicator to MPI_COMM_SELF preserves that expectation.
  MPI_Comm itsMpiComm = MPI_COMM_SELF;
#endif
};
 
}  // namespace casacore
 
#endif