DataMgr.h

Go to the documentation of this file.

#include <vector>
#include <iostream>
#include <list>
#include "vapor/VAssert.h"
#include <vapor/BlkMemMgr.h>
#include <vapor/DC.h>
#include <vapor/MyBase.h>
#include <vapor/RegularGrid.h>
#include <vapor/StretchedGrid.h>
#include <vapor/LayeredGrid.h>
#include <vapor/CurvilinearGrid.h>
#include <vapor/UnstructuredGrid2D.h>
#include <vapor/UDUnitsClass.h>
#include <vapor/GridHelper.h>
#include <vapor/DerivedVarMgr.h>
 
#ifndef DataMgvV3_0_h
    #define DataMgvV3_0_h
 
using namespace std;
 
namespace VAPoR {
class PipeLine;
class DerivedVar;
class DerivedDataVar;
class DerivedCoordVar;
 
//
class VDF_API DataMgr : public Wasp::MyBase {
public:
    //
    DataMgr(string format, size_t mem_size, int nthreads = 0);
 
    virtual ~DataMgr();
 
    //
    virtual int Initialize(const vector<string> &paths, const std::vector<string> &options);
 
    //
    std::vector<string> GetDimensionNames() const
    {
        VAssert(_dc);
        return (_dc->GetDimensionNames());
    }
 
    //
    bool GetAtt (string varname, string attname, vector< double > &values) const
    {
        VAssert(_dc);
        return (_dc->GetAtt(varname, attname, values));
    }
 
    //
    bool GetAtt (string varname, string attname, vector<long> &values) const
    {
        VAssert(_dc);
        return (_dc->GetAtt(varname, attname, values));
    }
 
    //
    bool GetAtt (string varname, string attname, string &values) const
    {
        VAssert(_dc);
        return (_dc->GetAtt(varname, attname, values));
    }
 
    //
    std::vector<string> GetAttNames(string varname) const
    {
        VAssert(_dc);
        return (_dc->GetAttNames(varname));
    }
 
    //
    DC::XType GetAttType(string varname, string attname) const
    {
        VAssert(_dc);
        return (_dc->GetAttType(varname, attname));
    }
 
    //
    bool GetDimension(string dimname, DC::Dimension &dimension, long ts) const
    {
        VAssert(_dc);
        return (_dc->GetDimension(dimname, dimension, ts));
    }
    
    long GetDimensionLength(string name, long ts) const
    {
        VAssert(_dc);
        DC::Dimension dimension;
        bool ok = GetDimension(name, dimension, ts);
        if (ok)
            return dimension.GetLength();
        return -1;
    }
 
    //
    std::vector<string> GetMeshNames() const
    {
        VAssert(_dc);
        return (_dc->GetMeshNames());
    }
 
    //
    bool GetMesh(string meshname, DC::Mesh &mesh) const;
 
    DC::Mesh GetMesh(string meshname) const {
        DC::Mesh mesh;
        GetMesh(meshname, mesh);
        return mesh;
    }
 
    virtual std::vector<string> GetDataVarNames() const;
 
    enum class VarType { Any, Scalar, Particle };
    virtual std::vector<string> GetDataVarNames(int ndim, VarType type = VarType::Any) const;
 
    virtual std::vector<string> GetCoordVarNames() const;
 
    void GetTimeCoordinates(std::vector<double> &timecoords) const { timecoords = _timeCoordinates; };
 
    const std::vector<double> &GetTimeCoordinates() const { return (_timeCoordinates); };
 
    //
    string GetTimeCoordVarName() const;
 
    bool HasTimeVaryingCoordinates() const;
    bool HasMovingDomain() const;
 
    //
    virtual bool GetVarCoordVars(string varname, bool spatial, std::vector<string> &coord_vars) const;
    vector<string> GetVarCoordVars(string varname, bool spatial) const;
 
    bool GetDataVarInfo(string varname, VAPoR::DC::DataVar &datavar) const;
 
    //
    bool GetBaseVarInfo(string varname, VAPoR::DC::BaseVar &var) const;
 
    bool GetCoordVarInfo(string varname, VAPoR::DC::CoordVar &cvar) const;
 
    bool IsTimeVarying(string varname) const;
 
    //
    bool IsCompressed(string varname) const;
 
    //
    int GetNumTimeSteps(string varname) const;
 
    //
    int GetNumTimeSteps() const;
 
    //
    size_t GetNumRefLevels(string varname) const;
 
    //
    std::vector<size_t> GetCRatios(string varname) const;
 
    //
    VAPoR::Grid *GetVariable(size_t ts, string varname, int level, int lod, bool lock = false);
 
    VAPoR::Grid *GetVariable(size_t ts, string varname, int level, int lod, CoordType min, CoordType max, bool lock = false);
 
    VAPoR::Grid *GetVariable(size_t ts, string varname, int level, int lod, DimsType min, DimsType max, bool lock = false);
 
    int GetVariableExtents(size_t ts, string varname, int level, int lod, CoordType &min, CoordType &max);
 
    //
    int GetDataRange(size_t ts, string varname, int level, int lod, std::vector<double> &range);
 
    //
    int GetDataRange(size_t ts, string varname, int level, int lod, CoordType min, CoordType max, std::vector<double> &range);
 
    virtual int GetDimLensAtLevel(string varname, int level, std::vector<size_t> &dims_at_level, long ts) const
    {
        std::vector<size_t> dummy;
        return (GetDimLensAtLevel(varname, level, dims_at_level, dummy, ts));
    }
 
    virtual int GetDimLens(string varname, std::vector<size_t> &dims, long ts) { return (GetDimLensAtLevel(varname, -1, dims, ts)); }
 
    std::vector<size_t> GetDimLens(string varname) {
        std::vector<size_t> dims;
        GetDimLens(varname, dims, 0);
        return dims;
    }
 
    //
    void UnlockGrid(const VAPoR::Grid *rg);
 
    //
    size_t GetNumDimensions(string varname) const;
 
    size_t GetVarTopologyDim(string varname) const;
 
    size_t GetVarGeometryDim(string varname) const;
 
    //
    void Clear();
 
    //
    virtual bool VariableExists(size_t ts, string varname, int level = 0, int lod = 0) const;
 
    //
    virtual string GetMapProjection() const { return (_proj4String); }
 
    //
    virtual string GetMapProjectionDefault() const { return (_proj4StringDefault); }
 
    #ifdef VAPOR3_0_0_ALPHA
 
    int NewPipeline(PipeLine *pipeline);
 
    void RemovePipeline(string name);
    #endif
 
    //
    bool IsVariableDerived(string varname) const;
 
    //
    bool IsVariableNative(string varname) const;
 
    int AddDerivedVar(DerivedDataVar *derivedVar);
 
    void RemoveDerivedVar(string varname);
 
    void PurgeVariable(string varname);
 
    class BlkExts {
    public:
        BlkExts(){};
        BlkExts(const DimsType &bmin, const DimsType &bmax);
 
        void Insert(const DimsType &bcoord, const CoordType &min, const CoordType &max);
 
        bool Intersect(const CoordType &min, const CoordType &max, DimsType &bmin, DimsType &bmax, int nCoords = 3) const;
 
        friend std::ostream &operator<<(std::ostream &o, const BlkExts &b);
 
    private:
        DimsType               _bmin = {{0, 0, 0}};
        DimsType               _bmax = {{0, 0, 0}};
        std::vector<CoordType> _mins;
        std::vector<CoordType> _maxs;
    };
 
protected:
    //
    // Cache for various metadata attributes
    //
    template<typename C> class VarInfoCache {
    public:
        //
        //
        void Set(size_t ts, std::vector<string> varnames, int level, int lod, string key, const std::vector<C> &values);
 
        void Set(size_t ts, string varname, int level, int lod, string key, const std::vector<C> &values)
        {
            std::vector<string> varnames;
            varnames.push_back(varname);
            Set(ts, varnames, level, lod, key, values);
        }
 
        bool Get(size_t ts, std::vector<string> varnames, int level, int lod, string key, std::vector<C> &values) const;
 
        bool Get(size_t ts, string varname, int level, int lod, string key, std::vector<C> &values) const
        {
            std::vector<string> varnames;
            varnames.push_back(varname);
            return Get(ts, varnames, level, lod, key, values);
        }
 
        void Purge(size_t ts, std::vector<string> varnames, int level, int lod, string key);
        void Purge(size_t ts, string varname, int level, int lod, string key)
        {
            std::vector<string> varnames;
            varnames.push_back(varname);
            Purge(ts, varnames, level, lod, key);
        }
        void Purge(std::vector<string> varnames);
 
        void Clear() { _cache.clear(); }
 
        static string _make_hash(string key, size_t ts, std::vector<string> cvars, int level, int lod);
 
        void _decode_hash(const string &hash, string &key, size_t &ts, vector<string> &varnames, int &level, int &lod);
 
    private:
        std::map<string, std::vector<C>> _cache;
    };
 
    mutable std::map<std::pair<VarType, size_t>, std::vector<string>> _dataVarNamesCache;
 
    string _format;
    int    _nthreads;
    size_t _mem_size;
 
    DC *              _dc;
    VAPoR::UDUnits    _udunits;
    VAPoR::GridHelper _gridHelper;
 
    DerivedVarMgr _dvm;
    bool          _doTransformHorizontal;
    bool          _doTransformVertical;
    string        _openVarName;
 
    std::vector<double> _timeCoordinates;
    string              _proj4String;
    string              _proj4StringDefault;
    DimsType            _bs;
 
    typedef struct {
        size_t              ts;
        string              varname;
        int                 level;
        int                 lod;
        DimsType            bmin;
        DimsType            bmax;
        int                 lock_counter;
        void *              blks;
    } region_t;
 
    // a list of all allocated regions
    std::list<region_t> _regionsList;
 
    VAPoR::BlkMemMgr *_blk_mem_mgr;
 
    std::vector<PipeLine *> _PipeLines;
 
    mutable VarInfoCache<size_t> _varInfoCacheSize_T;
    mutable VarInfoCache<double> _varInfoCacheDouble;
    mutable VarInfoCache<void *> _varInfoCacheVoidPtr;
 
    std::map<string, BlkExts> _blkExtsCache;
 
    std::map<const Grid *, vector<float *>> _lockedFloatBlks;
    std::map<const Grid *, vector<int *>>   _lockedIntBlks;
 
    // Get the immediate variable dependencies of a variable
    //
    std::vector<string> _get_var_dependencies_1(string varname) const;
 
    // Recursively get all of the dependencies of a list of variables.
    // Handles cycles in the dependency graph
    //
    std::vector<string> _get_var_dependencies_all(std::vector<string> varnames, std::vector<string> dependencies) const;
 
    // Return true if native data has a transformable horizontal coordinate
    //
    bool _hasHorizontalXForm() const;
 
    // Return true if native mesh has a transformable horizontal coordinate
    //
    bool _hasHorizontalXForm(string meshname) const;
 
    bool _get_coord_vars(string varname, std::vector<string> &scvars, string &tcvar) const;
 
    bool _get_coord_vars(string varname, vector<DC::CoordVar> &scvarsinfo, DC::CoordVar &tcvarinfo) const;
 
    int _initTimeCoord();
 
    int _get_default_projection(string &projection);
 
    VAPoR::RegularGrid *_make_grid_regular(const DimsType &dims, const std::vector<float *> &blkvec, DimsType &bs, DimsType &bmin, const DimsType &bmax) const;
 
    VAPoR::StretchedGrid *_make_grid_stretched(const DimsType &dims, const std::vector<float *> &blkvec, const DimsType &bs, const DimsType &bmin, const DimsType &bmax) const;
 
    VAPoR::LayeredGrid *_make_grid_layered(const DimsType &dims, const std::vector<float *> &blkvec, const DimsType &bs, const DimsType &bmin, const DimsType &bmax) const;
 
    VAPoR::CurvilinearGrid *_make_grid_curvilinear(size_t ts, int level, int lod, const vector<DC::CoordVar> &cvarsinfo, const DimsType &dims, const std::vector<float *> &blkvec, const DimsType &bs,
                                                   const DimsType &bmin, const DimsType &bmax);
 
    void _ugrid_setup(const DC::DataVar &var, DimsType &vertexDims, DimsType &faceDims, DimsType &edgeDims,
                      UnstructuredGrid::Location &location,    // node,face, edge
                      size_t &maxVertexPerFace, size_t &maxFacePerVertex, long &vertexOffset, long &faceOffset, long ts) const;
 
    UnstructuredGrid2D *_make_grid_unstructured2d(size_t ts, int level, int lod, const DC::DataVar &dvarinfo, const vector<DC::CoordVar> &cvarsinfo, const DimsType, const vector<float *> &blkvec,
                                                  const DimsType &bs, const DimsType &bmin, const DimsType &bmax, const vector<int *> &conn_blkvec, const DimsType &conn_bs, const DimsType &conn_bmin,
                                                  const DimsType &conn_bmax);
 
    VAPoR::Grid *_make_grid(size_t ts, int level, int lod, const VAPoR::DC::DataVar &var, const DimsType &roi_dims, const DimsType &dims, const std::vector<float *> &blkvec,
                            const std::vector<DimsType> &bsvec, const std::vector<DimsType> &bminvec, const std::vector<DimsType> &bmaxvec, const vector<int *> &conn_blkvec,
                            const vector<DimsType> &conn_bsvec, const vector<DimsType> &conn_bminvec, const vector<DimsType> &conn_bmaxvec);
 
    string _get_grid_type(string varname) const;
 
    int _find_bounding_grid(size_t ts, string varname, int level, int lod, CoordType min, CoordType max, DimsType &min_ui, DimsType &max_ui);
 
    void _setupCoordVecsHelper(string data_varname, const DimsType &data_dimlens, const DimsType &data_bmin, const DimsType &data_bmax, string coord_varname, int order, DimsType &coord_dimlens,
                               DimsType &coord_bmin, DimsType &coord_bmax, bool structured, long ts) const;
 
    int _setupCoordVecs(size_t ts, string varname, int level, int lod, const DimsType &min, const DimsType &max, vector<string> &varnames, DimsType &roi_dims, vector<DimsType> &dimsvec,
                        vector<DimsType> &bsvec, vector<DimsType> &bminvec, vector<DimsType> &bmaxvec, bool structured) const;
 
    int _setupConnVecs(size_t ts, string varname, int level, int lod, vector<string> &varnames, vector<DimsType> &dimsvec, vector<DimsType> &bsvec, vector<DimsType> &bminvec,
                       vector<DimsType> &bmaxvec) const;
 
    VAPoR::Grid *_getVariable(size_t ts, string varname, int level, int lod, bool lock, bool dataless);
 
    VAPoR::Grid *_getVariable(size_t ts, string varname, int level, int lod, DimsType min, DimsType max, bool lock, bool dataless);
 
    int _parseOptions(vector<string> &options);
 
    template<typename T> T *_get_region_from_cache(size_t ts, string varname, int level, int lod, const DimsType &bmin, const DimsType &bmax, bool lock);
 
    template<typename T>
    int _get_unblocked_region_from_fs(size_t ts, string varname, int level, int lod, const DimsType &grid_dims, const DimsType &grid_bs, const DimsType &grid_min, const DimsType &grid_max, T *blks);
 
    template<typename T>
    int _get_blocked_region_from_fs(size_t ts, string varname, int level, int lod, const DimsType &file_bs, const DimsType &file_dims, const DimsType &grid_dims, const DimsType &grid_bs,
                                    const DimsType &grid_min, const DimsType &grid_max, T *blks);
 
    template<typename T>
    T *_get_region_from_fs(size_t ts, string varname, int level, int lod, const DimsType &grid_dims, const DimsType &grid_bs, const DimsType &grid_bmin, const DimsType &grid_bmax, bool lock);
 
    template<typename T> T *_get_region(size_t ts, string varname, int level, int lod, int nlods, const DimsType &dims, const DimsType &bs, const DimsType &bmin, const DimsType &bmax, bool lock);
 
    template<typename T>
    int _get_regions(size_t ts, const std::vector<string> &varnames, int level, int lod, bool lock, const std::vector<DimsType> &dimsvec, const std::vector<DimsType> &bsvec,
                     const std::vector<DimsType> &bminvec, const std::vector<DimsType> &bmaxvec, std::vector<T *> &blkvec);
 
    void _unlock_blocks(const void *blks);
 
    std::vector<string> _get_native_variables() const;
 
    void *_alloc_region(size_t ts, string varname, int level, int lod, DimsType bmin, DimsType bmax, DimsType bs, int element_sz, bool lock, bool fill);
 
    void _free_region(size_t ts, string varname, int level, int lod, DimsType bmin, DimsType bmax, bool forceFlag = false);
 
    bool _free_lru();
    void _free_var(string varname);
 
    int _level_correction(string varname, int &level) const;
    int _lod_correction(string varname, int &lod) const;
 
    vector<string> _getDataVarNamesDerived(int ndim) const;
 
    bool _hasCoordForAxis(vector<string> coord_vars, int axis) const;
 
    string _defaultCoordVar(const DC::Mesh &m, int axis) const;
 
    void _assignHorizontalCoords(vector<string> &coord_vars) const;
 
    bool _getVarDimensions(string varname, vector<DC::Dimension> &dimensions, long ts) const;
 
    bool _getDataVarDimensions(string varname, vector<DC::Dimension> &dimensions, long ts) const;
 
    bool _getCoordVarDimensions(string varname, vector<DC::Dimension> &dimensions, long ts) const;
 
    bool _getVarDimNames(string varname, vector<string> &dimnames) const;
 
    bool _isDataVar(string varname) const
    {
        vector<string> names = GetDataVarNames();
        return (find(names.begin(), names.end(), varname) != names.end());
    }
 
    bool _isCoordVar(string varname) const
    {
        vector<string> names = GetCoordVarNames();
        return (find(names.begin(), names.end(), varname) != names.end());
    }
 
    bool _getVarConnVars(string varname, string &face_node_var, string &node_face_var, string &face_edge_var, string &face_face_var, string &edge_node_var, string &edge_face_var) const;
 
    DerivedVar *     _getDerivedVar(string varname) const;
    DerivedDataVar * _getDerivedDataVar(string varname) const;
    DerivedCoordVar *_getDerivedCoordVar(string varname) const;
 
    int _openVariableRead(size_t ts, string varname, int level, int lod);
 
    template<class T> int _readRegion(int fd, const DimsType &min, const DimsType &max, size_t ndims, T *region);
    int                   _closeVariable(int fd);
 
    template<class T> int _getVar(string varname, int level, int lod, T *data);
 
    template<class T> int _getVar(size_t ts, string varname, int level, int lod, T *data);
 
    void _getLonExtents(std::vector<float> &lons, DimsType dims, float &min, float &max) const;
 
    void _getLatExtents(std::vector<float> &lons, DimsType dims, float &min, float &max) const;
 
    int _getCoordPairExtents(string lon, string lat, float &lonmin, float &lonmax, float &latmin, float &latmax, long ts);
 
    int _initProj4StringDefault();
 
    int _initHorizontalCoordVars();
 
    int _initVerticalCoordVars();
 
    bool _hasVerticalXForm() const;
 
    bool _hasVerticalXForm(string meshname, string &standard_name, string &formula_terms) const;
 
    bool _hasVerticalXForm(string meshname) const
    {
        string standard_name, formula_terms;
        return (_hasVerticalXForm(meshname, standard_name, formula_terms));
    }
 
    bool _isCoordVarInUse(string varName) const;
 
    // Hide public DC::GetDimLensAtLevel by making it private
    //
    virtual int GetDimLensAtLevel(string varname, int level, std::vector<size_t> &dims_at_level, std::vector<size_t> &bs_at_level, long ts) const;
};
 
};    // namespace VAPoR
#endif