pio.h

Go to the documentation of this file.

 
#ifndef _PIO_H_
#define _PIO_H_
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h> /* memcpy */
#include <mpi.h>
#include <uthash.h>
 
#include <netcdf.h>
 
#define PIO_OFFSET MPI_OFFSET
 
#define PIO_Offset MPI_Offset
 
#define PIO_MAX_VARS NC_MAX_VARS
 
#define PIO_MAX_DIMS NC_MAX_DIMS
 
#define PIO_DEFAULT (-1)
 
#define DECOMP_VERSION_ATT_NAME "PIO_library_version"
 
#define DECOMP_MAX_MAPLEN_ATT_NAME "max_maplen"
 
#define DECOMP_TITLE_ATT_NAME "title"
 
#define DECOMP_HISTORY_ATT_NAME "history"
 
#define DECOMP_SOURCE_ATT_NAME "source"
 
#define DECOMP_ORDER_ATT_NAME "array_order"
 
#define DECOMP_BACKTRACE_ATT_NAME "backtrace"
 
#define DECOMP_DIM_DIM "dims"
 
#define DECOMP_TASK_DIM_NAME "task"
 
#define DECOMP_MAPELEM_DIM_NAME "map_element"
 
#define DECOMP_NDIMS "ndims"
 
#define DECOMP_GLOBAL_SIZE_VAR_NAME "global_size"
 
#define DECOMP_MAPLEN_VAR_NAME "maplen"
 
#define DECOMP_MAP_VAR_NAME "map"
 
#define DECOMP_C_ORDER_STR "C"
 
#define DECOMP_FORTRAN_ORDER_STR "Fortran"
 
#define NC_PIO NC_UDF0
 
typedef struct var_desc_t
{
    int varid;
 
    int rec_var;
 
    int record;
 
    int *request;
 
    int nreqs;
 
    void *fillvalue;
 
    int ndims;
 
    int use_fill;
 
    void *fillbuf;
 
    int pio_type;
 
    int pio_type_size;
 
    MPI_Datatype mpi_type;
 
    int mpi_type_size;
 
    UT_hash_handle hh;
 
} var_desc_t;
 
typedef struct io_region
{
    int loffset;
 
    PIO_Offset *start;
 
    PIO_Offset *count;
 
    struct io_region *next;
} io_region;
 
enum PIO_REARR_COMM_TYPE
{
    PIO_REARR_COMM_P2P = (0),
 
    PIO_REARR_COMM_COLL
};
 
enum PIO_REARR_COMM_FC_DIR
{
    PIO_REARR_COMM_FC_2D_ENABLE = (0),
 
    PIO_REARR_COMM_FC_1D_COMP2IO,
 
    PIO_REARR_COMM_FC_1D_IO2COMP,
 
    PIO_REARR_COMM_FC_2D_DISABLE
};
 
#define PIO_REARR_COMM_UNLIMITED_PEND_REQ -1
 
typedef struct rearr_comm_fc_opt
{
    bool hs;
 
    bool isend;
 
    int max_pend_req;
} rearr_comm_fc_opt_t;
 
typedef struct rearr_opt
{
    int comm_type;
 
    int fcd;
 
    rearr_comm_fc_opt_t comp2io;
 
    rearr_comm_fc_opt_t io2comp;
} rearr_opt_t;
 
typedef struct io_desc_t
{
    int ioid;
 
    int maplen;
 
    PIO_Offset *map;
 
    int *remap;
 
    int nrecvs;
 
    int ndof;
 
    int ndims;
 
    int *dimlen;
 
    int num_aiotasks;
 
    int rearranger;
 
    int maxregions;
 
    bool needsfill;
 
    bool needssort;
 
    int maxbytes;
 
    int piotype;
 
    int piotype_size;
 
    MPI_Datatype mpitype;
 
    int mpitype_size;
 
    PIO_Offset llen;
 
    int maxiobuflen;
 
    int *rfrom;
 
    int *rcount;
 
    int *scount;
 
    PIO_Offset *sindex;
 
    PIO_Offset *rindex;
 
    MPI_Datatype *rtype;
 
    MPI_Datatype *stype;
 
    int num_stypes;
 
    int holegridsize;
 
    int maxholegridsize;
 
    int maxfillregions;
 
    io_region *firstregion;
 
    io_region *fillregion;
 
    rearr_opt_t rearr_opts;
 
    MPI_Comm subset_comm;
 
    UT_hash_handle hh;
 
} io_desc_t;
 
typedef struct iosystem_desc_t
{
    int iosysid;
 
    MPI_Comm union_comm;
 
    MPI_Comm io_comm;
 
    MPI_Comm comp_comm;
 
    MPI_Comm intercomm;
 
    MPI_Comm my_comm;
 
    int num_iotasks;
 
    int num_comptasks;
 
    int num_uniontasks;
 
    int union_rank;
 
    int comp_rank;
 
    int io_rank;
 
    int iomaster;
 
    int compmaster;
 
    int ioroot;
 
    int comproot;
 
    int *ioranks;
 
    int *compranks;
 
    int error_handler;
 
    int default_rearranger;
 
    bool async;
 
    bool ioproc;
 
    bool compproc;
 
    MPI_Info info;
 
    int comp_idx;
 
    rearr_opt_t rearr_opts;
 
    struct iosystem_desc_t *next;
} iosystem_desc_t;
 
typedef struct wmulti_buffer
{
    int ioid;
 
    int recordvar;
 
    int num_arrays;
 
    int arraylen;
 
    int *vid;
 
    int *frame;
 
    void *fillvalue;
 
    void *data;
 
    int htid;
 
    UT_hash_handle hh;
} wmulti_buffer;
 
typedef struct file_desc_t
{
    iosystem_desc_t *iosystem;
 
    int fh;
 
    int pio_ncid;
 
    int iotype;
 
    struct var_desc_t *varlist;
 
    int nvars;
 
    int writable;
 
    struct wmulti_buffer *buffer;
 
    void *iobuf;
 
    int pio_type;
 
    UT_hash_handle hh;
 
    int do_io;
 
    int ncint_file;
} file_desc_t;
 
enum PIO_IOTYPE
{
    PIO_IOTYPE_PNETCDF = 1,
 
    PIO_IOTYPE_NETCDF = 2,
 
    PIO_IOTYPE_NETCDF4C = 3,
 
    PIO_IOTYPE_NETCDF4P = 4
};
 
enum PIO_REARRANGERS
{
    PIO_REARR_BOX = 1,
 
    PIO_REARR_SUBSET = 2
};
 
enum PIO_ERROR_HANDLERS
{
    PIO_INTERNAL_ERROR = (-51),
 
    PIO_BCAST_ERROR = (-52),
 
    PIO_RETURN_ERROR = (-53)
};
 
#define PIO_GLOBAL NC_GLOBAL
 
#define PIO_UNLIMITED NC_UNLIMITED
 
/* NetCDF types. */
#define PIO_BYTE   NC_BYTE       
#define PIO_CHAR   NC_CHAR       
#define PIO_SHORT  NC_SHORT      
#define PIO_INT    NC_INT        
#define PIO_FLOAT  NC_FLOAT      
#define PIO_REAL   NC_FLOAT      
#define PIO_DOUBLE NC_DOUBLE     
#define PIO_UBYTE  NC_UBYTE      
#define PIO_USHORT NC_USHORT     
#define PIO_UINT   NC_UINT       
#define PIO_INT64  NC_INT64      
#define PIO_UINT64 NC_UINT64     
#define PIO_STRING NC_STRING     
/* NetCDF flags. */
#define PIO_WRITE  NC_WRITE      
#define PIO_NOWRITE  NC_NOWRITE  
#define PIO_CLOBBER NC_CLOBBER   
#define PIO_NOCLOBBER NC_NOCLOBBER  
#define PIO_FILL NC_FILL         
#define PIO_NOFILL NC_NOFILL     
#define PIO_MAX_NAME NC_MAX_NAME 
#define PIO_MAX_VAR_DIMS NC_MAX_VAR_DIMS 
#define PIO_64BIT_OFFSET NC_64BIT_OFFSET 
#define PIO_64BIT_DATA 0x0010 
#define PIO_NOERR  NC_NOERR           
#define PIO_EBADID NC_EBADID          
#define PIO_ENFILE NC_ENFILE          
#define PIO_EEXIST NC_EEXIST          
#define PIO_EINVAL NC_EINVAL          
#define PIO_EPERM NC_EPERM            
#define PIO_ENOTINDEFINE NC_ENOTINDEFINE 
#define PIO_EINDEFINE NC_EINDEFINE    
#define PIO_EINVALCOORDS NC_EINVALCOORDS 
#define PIO_EMAXDIMS NC_EMAXDIMS      
#define PIO_ENAMEINUSE NC_ENAMEINUSE  
#define PIO_ENOTATT NC_ENOTATT        
#define PIO_EMAXATTS NC_EMAXATTS      
#define PIO_EBADTYPE NC_EBADTYPE      
#define PIO_EBADDIM NC_EBADDIM        
#define PIO_EUNLIMPOS NC_EUNLIMPOS    
#define PIO_EMAXVARS NC_EMAXVARS      
#define PIO_ENOTVAR NC_ENOTVAR        
#define PIO_EGLOBAL NC_EGLOBAL        
#define PIO_ENOTNC NC_ENOTNC          
#define PIO_ESTS NC_ESTS              
#define PIO_EMAXNAME NC_EMAXNAME      
#define PIO_EUNLIMIT NC_EUNLIMIT      
#define PIO_ENORECVARS NC_ENORECVARS  
#define PIO_ECHAR NC_ECHAR            
#define PIO_EEDGE NC_EEDGE            
#define PIO_ESTRIDE NC_ESTRIDE        
#define PIO_EBADNAME NC_EBADNAME      
#define PIO_ERANGE NC_ERANGE          
#define PIO_ENOMEM NC_ENOMEM          
#define PIO_EVARSIZE NC_EVARSIZE      
#define PIO_EDIMSIZE NC_EDIMSIZE      
#define PIO_ETRUNC NC_ETRUNC          
#define PIO_EAXISTYPE NC_EAXISTYPE    
#define PIO_EDAP NC_EDAP              
#define PIO_ECURL NC_ECURL            
#define PIO_EIO NC_EIO                
#define PIO_ENODATA NC_ENODATA        
#define PIO_EDAPSVC NC_EDAPSVC        
#define PIO_EDAS NC_EDAS              
#define PIO_EDDS NC_EDDS              
#define PIO_EDATADDS NC_EDATADDSDS    
#define PIO_EDAPURL NC_EDAPURL        
#define PIO_EDAPCONSTRAINT NC_EDAPCONSTRAINT 
#define PIO_ETRANSLATION NC_ETRANSLATION     
#define PIO_EHDFERR NC_EHDFERR        
#define PIO_ECANTREAD NC_ECANTREAD    
#define PIO_ECANTWRITE NC_ECANTWRITE  
#define PIO_ECANTCREATE NC_ECANTCREATE 
#define PIO_EFILEMETA NC_EFILEMETA    
#define PIO_EDIMMETA NC_EDIMMETA      
#define PIO_EATTMETA NC_EATTMETA      
#define PIO_EVARMETA NC_EVARMETA      
#define PIO_ENOCOMPOUND NC_ENOCOMPOUND  
#define PIO_EATTEXISTS NC_EATTEXISTS  
#define PIO_ENOTNC4 NC_ENOTNC4        
#define PIO_ESTRICTNC3 NC_ESTRICTNC3  
#define PIO_ENOTNC3 NC_ENOTNC3        
#define PIO_ENOPAR NC_ENOPAR          
#define PIO_EPARINIT NC_EPARINIT      
#define PIO_EBADGRPID NC_EBADGRPID    
#define PIO_EBADTYPID NC_EBADTYPID    
#define PIO_ETYPDEFINED NC_ETYPDEFINED    
#define PIO_EBADFIELD NC_EBADFIELD    
#define PIO_EBADCLASS NC_EBADCLASS    
#define PIO_EMAPTYPE NC_EMAPTYPE      
#define PIO_ELATEFILL NC_ELATEFILL    
#define PIO_ELATEDEF NC_ELATEDEF      
#define PIO_EDIMSCALE NC_EDIMSCALE    
#define PIO_ENOGRP NC_ENOGRP          
#define PIO_ESTORAGE NC_ESTORAGE      
#define PIO_EBADCHUNK NC_EBADCHUNK    
#define PIO_ENOTBUILT NC_ENOTBUILT    
#define PIO_EDISKLESS NC_EDISKLESS    
/* These are the netCDF default fill values. */
#define PIO_FILL_BYTE NC_FILL_BYTE     
#define PIO_FILL_CHAR NC_FILL_CHAR     
#define PIO_FILL_SHORT NC_FILL_SHORT   
#define PIO_FILL_INT NC_FILL_INT       
#define PIO_FILL_FLOAT NC_FILL_FLOAT   
#define PIO_FILL_DOUBLE NC_FILL_DOUBLE 
#define PIO_FILL_UBYTE NC_FILL_UBYTE   
#define PIO_FILL_USHORT NC_FILL_USHORT 
#define PIO_FILL_UINT NC_FILL_UINT     
#define PIO_FILL_INT64 NC_FILL_INT64   
#define PIO_FILL_UINT64 NC_FILL_UINT64 
#define PIO_EINDEP  (-203)  
#define PIO_FIRST_ERROR_CODE (-500)  
#define PIO_EBADIOTYPE  (-500)       
#define PIO_EVARDIMMISMATCH (-501)   
#define PIO_EBADREARR (-502)        
#define PIO_REQ_NULL (NC_REQ_NULL-1) 
#if defined(__cplusplus)
extern "C" {
#endif
    /* Error handling. */
    int PIOc_strerror(int pioerr, char *errstr);
    int PIOc_set_log_level(int level);
    int PIOc_set_global_log_level(int iosysid, int level);
 
    /* Decomposition. */
 
    /* Init decomposition with 1-based compmap array. */
    int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen,
                        const PIO_Offset *compmap, int *ioidp, const int *rearr,
                        const PIO_Offset *iostart, const PIO_Offset *iocount);
    int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *gdimlen,
                           const long int *start, const long int *count, int *ioidp);
 
    /* Init decomposition with 0-based compmap array. */
    int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen,
                         const PIO_Offset *compmap, int *ioidp, int rearranger,
                         const PIO_Offset *iostart, const PIO_Offset *iocount);
 
    /* Free resources associated with a decomposition. */
    int PIOc_freedecomp(int iosysid, int ioid);
 
    int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen,
                     PIO_Offset **map, MPI_Comm comm);
    int PIOc_readmap_from_f90(const char *file,int *ndims, int **gdims, PIO_Offset *maplen,
                              PIO_Offset **map, int f90_comm);
    int PIOc_writemap(const char *file, int ndims, const int *gdims, PIO_Offset maplen,
                      PIO_Offset *map, MPI_Comm comm);
    int PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims,
                               PIO_Offset maplen, const PIO_Offset *map, int f90_comm);
 
    /* Write a decomposition file. */
    int PIOc_write_decomp(const char *file, int iosysid, int ioid, MPI_Comm comm);
 
    /* Write a decomposition file using netCDF. */
    int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid,
                             char *title, char *history, int fortran_order);
 
    /* Read a netCDF decomposition file. */
    int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioid, MPI_Comm comm,
                            int pio_type, char *title, char *history, int *fortran_order);
 
    /* Initializing IO system for async. */
    int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, int component_count,
                        int *num_procs_per_comp, int **proc_list, MPI_Comm *io_comm, MPI_Comm *comp_comm,
                        int rearranger, int *iosysidp);
 
    /* Initializing IO system for async - alternative interface. */
    int PIOc_init_async_from_comms(MPI_Comm world, int component_count, MPI_Comm *comp_comm,
                                   MPI_Comm io_comm, int rearranger, int *iosysidp);
 
    /* How many IO tasks in this iosysid? */
    int PIOc_get_numiotasks(int iosysid, int *numiotasks);
 
    /* Initialize PIO for intracomm mode. */
    int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int rearr,
                            int *iosysidp);
 
    int PIOc_finalize(int iosysid);
 
    /* Shut down an iosystem and free all associated resources. */
    int PIOc_free_iosystem(int iosysid);
 
    /* Set error handling for entire io system. */
    int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method);
 
    /* Set error handling for entire io system. */
    int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method);
 
    /* Determine whether this is IO task. */
    int PIOc_iam_iotask(int iosysid, bool *ioproc);
 
    /* What is the iorank? */
    int PIOc_iotask_rank(int iosysid, int *iorank);
 
    /* Is this iosystem active? */
    int PIOc_iosystem_is_active(int iosysid, bool *active);
 
    /* Is this IOTYPE available? */
    int PIOc_iotype_available(int iotype);
 
    /* Set the options for the rearranger. */
    int PIOc_set_rearr_opts(int iosysid, int comm_type, int fcd,
                            bool enable_hs_c2i, bool enable_isend_c2i,
                            int max_pend_req_c2i,
                            bool enable_hs_i2c, bool enable_isend_i2c,
                            int max_pend_req_i2c);
 
    /* Increment record number. */
    int PIOc_advanceframe(int ncid, int varid);
 
    /* Set the record number. */
    int PIOc_setframe(int ncid, int varid, int frame);
 
    /* Write a distributed array. */
    int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array,
                          void *fillvalue);
 
    /* Write multiple darrays. */
    int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PIO_Offset arraylen,
                                void *array, const int *frame, void **fillvalue, bool flushtodisk);
 
    /* Read distributed array. */
    int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array);
 
    /* Get size of local distributed array. */
    int PIOc_get_local_array_size(int ioid);
 
    /* Handling files. */
    int PIOc_redef(int ncid);
    int PIOc_enddef(int ncid);
    int PIOc_sync(int ncid);
    int PIOc_deletefile(int iosysid, const char *filename);
    int PIOc_createfile(int iosysid, int *ncidp,  int *iotype, const char *fname, int mode);
    int PIOc_create(int iosysid, const char *path, int cmode, int *ncidp);
    int PIOc_openfile(int iosysid, int *ncidp, int *iotype, const char *fname, int mode);
    int PIOc_openfile2(int iosysid, int *ncidp, int *iotype, const char *fname, int mode);
    int PIOc_open(int iosysid, const char *path, int mode, int *ncidp);
    int PIOc_closefile(int ncid);
    int PIOc_inq_format(int ncid, int *formatp);
    int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp);
    int PIOc_inq_ndims(int ncid, int *ndimsp);
    int PIOc_inq_nvars(int ncid, int *nvarsp);
    int PIOc_inq_natts(int ncid, int *ngattsp);
    int PIOc_inq_unlimdim(int ncid, int *unlimdimidp);
    int PIOc_inq_unlimdims(int ncid, int *nunlimdimsp, int *unlimdimidsp);
    int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep);
    int PIOc_set_blocksize(int newblocksize);
    int PIOc_File_is_Open(int ncid);
 
    /* Set the IO node data buffer size limit. */
    PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit);
 
    /* Set the error hanlding for a file. */
    int PIOc_Set_File_Error_Handling(int ncid, int method);
 
    int PIOc_set_hint(int iosysid, const char *hint, const char *hintval);
    int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset nelems,
                             float preemption);
    int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset *nelemsp,
                             float *preemptionp);
 
    /* Dimensions. */
    int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp);
    int PIOc_inq_dimid(int ncid, const char *name, int *idp);
    int PIOc_inq_dimname(int ncid, int dimid, char *name);
    int PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp);
    int PIOc_rename_dim(int ncid, int dimid, const char *name);
    int PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp);
 
    /* Variables. */
    int PIOc_inq_varid(int ncid, const char *name, int *varidp);
    int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp,
                     int *nattsp);
    int PIOc_inq_varname(int ncid, int varid, char *name);
    int PIOc_inq_vartype(int ncid, int varid, nc_type *xtypep);
    int PIOc_inq_varndims(int ncid, int varid, int *ndimsp);
    int PIOc_inq_vardimid(int ncid, int varid, int *dimidsp);
    int PIOc_inq_varnatts(int ncid, int varid, int *nattsp);
    int PIOc_def_var(int ncid, const char *name, nc_type xtype,  int ndims,
                     const int *dimidsp, int *varidp);
    int PIOc_set_fill(int ncid, int fillmode, int *old_modep);
    int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value);
    int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep);
    int PIOc_rename_var(int ncid, int varid, const char *name);
 
    /* These variable settings only apply to netCDF-4 files. */
    int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
                             int deflate_level);
    int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep,
                             int *deflate_levelp);
    int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp);
    int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp);
    int PIOc_def_var_endian(int ncid, int varid, int endian);
    int PIOc_inq_var_endian(int ncid, int varid, int *endianp);
    int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems,
                                 float preemption);
    int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp,
                                 float *preemptionp);
 
    /* Attributes - misc. */
    int PIOc_rename_att(int ncid, int varid, const char *name, const char *newname);
    int PIOc_del_att(int ncid, int varid, const char *name);
 
    /* Attributes - inquiry functions. */
    int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep,
                     PIO_Offset *lenp);
    int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp);
    int PIOc_inq_attlen(int ncid, int varid, const char *name, PIO_Offset *lenp);
    int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep);
    int PIOc_inq_attname(int ncid, int varid, int attnum, char *name);
 
    /* Attributes - writing. */
    int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const void *op);
    int PIOc_put_att_text(int ncid, int varid, const char *name, PIO_Offset len, const char *op);
    int PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
                           const signed char *op);
    int PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
                           const short *op);
    int PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
                         const int *op);
    int PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
                          const long *op);
    int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
                           const float *op);
    int PIOc_put_att_double(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
                            const double *op);
    int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
                           const unsigned char *op);
    int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
                            const unsigned short *op);
    int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
                          const unsigned int *op);
    int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
                              const long long *op);
    int PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype,
                               PIO_Offset len, const unsigned long long *op);
 
    /* Attributes - reading. */
    int PIOc_get_att(int ncid, int varid, const char *name, void *ip);
    int PIOc_get_att_text(int ncid, int varid, const char *name, char *ip);
    int PIOc_get_att_schar(int ncid, int varid, const char *name, signed char *ip);
    int PIOc_get_att_short(int ncid, int varid, const char *name, short *ip);
    int PIOc_get_att_int(int ncid, int varid, const char *name, int *ip);
    int PIOc_get_att_long(int ncid, int varid, const char *name, long *ip);
    int PIOc_get_att_float(int ncid, int varid, const char *name, float *ip);
    int PIOc_get_att_double(int ncid, int varid, const char *name, double *ip);
    int PIOc_get_att_uchar(int ncid, int varid, const char *name, unsigned char *ip);
    int PIOc_get_att_ushort(int ncid, int varid, const char *name, unsigned short *ip);
    int PIOc_get_att_uint(int ncid, int varid, const char *name, unsigned int *ip);
    int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip);
    int PIOc_get_att_ulonglong(int ncid, int varid, const char *name, unsigned long long *ip);
 
    /* Data reads - var. */
    int PIOc_get_var(int ncid, int varid, void *buf);
    int PIOc_get_var_text(int ncid, int varid, char *buf);
    int PIOc_get_var_schar(int ncid, int varid, signed char *buf);
    int PIOc_get_var_short(int ncid, int varid, short *buf);
    int PIOc_get_var_int(int ncid, int varid, int *buf);
    int PIOc_get_var_long(int ncid, int varid, long *buf);
    int PIOc_get_var_float(int ncid, int varid, float *buf);
    int PIOc_get_var_double(int ncid, int varid, double *buf);
    int PIOc_get_var_uchar(int ncid, int varid, unsigned char *buf);
    int PIOc_get_var_ushort(int ncid, int varid, unsigned short *buf);
    int PIOc_get_var_uint(int ncid, int varid, unsigned int *buf);
    int PIOc_get_var_longlong(int ncid, int varid, long long *buf);
    int PIOc_get_var_ulonglong(int ncid, int varid, unsigned long long *buf);
 
    /* Data writes - var. */
    int PIOc_put_var(int ncid, int varid, const void *buf);
    int PIOc_put_var_text(int ncid, int varid, const char *op);
    int PIOc_put_var_schar(int ncid, int varid, const signed char *op);
    int PIOc_put_var_short(int ncid, int varid, const short *op);
    int PIOc_put_var_int(int ncid, int varid, const int *op);
    int PIOc_put_var_long(int ncid, int varid, const long *op);
    int PIOc_put_var_float(int ncid, int varid, const float *op);
    int PIOc_put_var_double(int ncid, int varid, const double *op);
    int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op);
    int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op);
    int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op);
    int PIOc_put_var_longlong(int ncid, int varid, const long long *op);
    int PIOc_put_var_ulonglong(int ncid, int varid, const unsigned long long *op);
 
    /* Data reads - var1. */
    int PIOc_get_var1(int ncid, int varid, const PIO_Offset *index, void *buf);
    int PIOc_get_var1_text(int ncid, int varid, const PIO_Offset *index, char *buf);
    int PIOc_get_var1_schar(int ncid, int varid, const PIO_Offset *index, signed char *buf);
    int PIOc_get_var1_short(int ncid, int varid, const PIO_Offset *index, short *buf);
    int PIOc_get_var1_int(int ncid, int varid, const PIO_Offset *index, int *buf);
    int PIOc_get_var1_long(int ncid, int varid, const PIO_Offset *index, long *buf);
    int PIOc_get_var1_float(int ncid, int varid, const PIO_Offset *index, float *buf);
    int PIOc_get_var1_double(int ncid, int varid, const PIO_Offset *index, double *buf);
    int PIOc_get_var1_uchar(int ncid, int varid, const PIO_Offset *index, unsigned char *buf);
    int PIOc_get_var1_ushort(int ncid, int varid, const PIO_Offset *index, unsigned short *buf);
    int PIOc_get_var1_uint(int ncid, int varid, const PIO_Offset *index, unsigned int *buf);
    int PIOc_get_var1_longlong(int ncid, int varid, const PIO_Offset *index, long long *buf);
    int PIOc_get_var1_ulonglong(int ncid, int varid, const PIO_Offset *index, unsigned long long *buf);
 
    /* Data writes - var1. */
    int PIOc_put_var1(int ncid, int varid, const PIO_Offset *index, const void *buf);
    int PIOc_put_var1_text(int ncid, int varid, const PIO_Offset *index, const char *op);
    int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset *index, const signed char *op);
    int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset *index, const short *op);
    int PIOc_put_var1_int(int ncid, int varid, const PIO_Offset *index, const int *op);
    int PIOc_put_var1_long(int ncid, int varid, const PIO_Offset *index, const long *ip);
    int PIOc_put_var1_float(int ncid, int varid, const PIO_Offset *index, const float *op);
    int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset *index, const double *op);
    int PIOc_put_var1_uchar(int ncid, int varid, const PIO_Offset *index,
                            const unsigned char *op);
    int PIOc_put_var1_ushort(int ncid, int varid, const PIO_Offset *index,
                             const unsigned short *op);
    int PIOc_put_var1_uint(int ncid, int varid, const PIO_Offset *index,
                           const unsigned int *op);
    int PIOc_put_var1_longlong(int ncid, int varid, const PIO_Offset *index, const long long *op);
    int PIOc_put_var1_ulonglong(int ncid, int varid, const PIO_Offset *index,
                                const unsigned long long *op);
 
    /* Data reads - vara. */
    int PIOc_get_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, void *buf);
    int PIOc_get_vara_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                           char *buf);
    int PIOc_get_vara_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                            signed char *buf);
    int PIOc_get_vara_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                            short *buf);
    int PIOc_get_vara_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                          int *buf);
    int PIOc_get_vara_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                            float *buf);
    int PIOc_get_vara_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                           long *buf);
    int PIOc_get_vara_double(int ncid, int varid, const PIO_Offset *start,
                             const PIO_Offset *count, double *buf);
    int PIOc_get_vara_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                            unsigned char *buf);
    int PIOc_get_vara_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                             unsigned short *buf);
    int PIOc_get_vara_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                           unsigned int *buf);
    int PIOc_get_vara_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                               long long *buf);
    int PIOc_get_vara_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                                unsigned long long *buf);
 
    /* Data writes - vara. */
    int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                      const void *buf);
    int PIOc_put_vara_text(int ncid, int varid, const PIO_Offset *start,
                           const PIO_Offset *count, const char *op);
    int PIOc_put_vara_schar(int ncid, int varid, const PIO_Offset *start,
                            const PIO_Offset *count, const signed char *op);
    int PIOc_put_vara_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                            const short *op);
    int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                          const int *op);
    int PIOc_put_vara_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                           const long *op);
    int PIOc_put_vara_float(int ncid, int varid, const PIO_Offset *start,
                            const PIO_Offset *count, const float *op);
    int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                             const double *op);
    int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                            const unsigned char *op);
    int PIOc_put_vara_ushort(int ncid, int varid, const PIO_Offset *start,
                             const PIO_Offset *count, const unsigned short *op);
    int PIOc_put_vara_uint(int ncid, int varid, const PIO_Offset *start,
                           const PIO_Offset *count, const unsigned int *op);
    int PIOc_put_vara_longlong(int ncid, int varid, const PIO_Offset *start,
                               const PIO_Offset *count, const long long *op);
    int PIOc_put_vara_ulonglong(int ncid, int varid, const PIO_Offset *start,
                                const PIO_Offset *count, const unsigned long long *op);
 
    /* Data reads - vars. */
    int PIOc_get_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                      const PIO_Offset *stride, void *buf);
    int PIOc_get_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                           const PIO_Offset *stride, char *buf);
    int PIOc_get_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                            const PIO_Offset *stride, signed char *buf);
    int PIOc_get_vars_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                            const PIO_Offset *stride, short *buf);
    int PIOc_get_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                          const PIO_Offset *stride, int *buf);
    int PIOc_get_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                           const PIO_Offset *stride, long *buf);
    int PIOc_get_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                            const PIO_Offset *stride, float *buf);
    int PIOc_get_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                             const PIO_Offset *stride, double *buf);
    int PIOc_get_vars_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                            const PIO_Offset *stride, unsigned char *buf);
    int PIOc_get_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                             const PIO_Offset *stride, unsigned short *buf);
    int PIOc_get_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                           const PIO_Offset *stride, unsigned int *buf);
    int PIOc_get_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                               const PIO_Offset *stride, long long *buf);
    int PIOc_get_vars_ulonglong(int ncid, int varid, const PIO_Offset *start,
                                const PIO_Offset *count, const PIO_Offset *stride,
                                unsigned long long *buf);
 
    /* Data writes - vars. */
    int PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                      const PIO_Offset *stride, const void *buf);
    int PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start,
                           const PIO_Offset *count, const PIO_Offset *stride, const char *op);
    int PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start,
                            const PIO_Offset *count, const PIO_Offset *stride,
                            const signed char *op);
    int PIOc_put_vars_short(int ncid, int varid, const PIO_Offset *start,
                            const PIO_Offset *count, const PIO_Offset *stride, const short *op);
    int PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                          const PIO_Offset *stride, const int *op);
    int PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                            const PIO_Offset *stride, const float *op);
    int PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start,
                             const PIO_Offset *count, const PIO_Offset *stride, const double *op);
    int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                           const PIO_Offset *stride, const long *op);
    int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                            const PIO_Offset *stride, const unsigned char *op);
    int PIOc_put_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                             const PIO_Offset *stride, const unsigned short *op);
    int PIOc_put_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                           const PIO_Offset *stride, const unsigned int *op);
    int PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
                               const PIO_Offset *stride, const long long *op);
    int PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset *start,
                                const PIO_Offset *count, const PIO_Offset *stride,
                                const unsigned long long *op);
 
    /* Data reads - vard. */
    int PIOc_get_vard(int ncid, int varid, int decompid, const PIO_Offset recnum, void *buf);
    int PIOc_get_vard_text(int ncid, int varid, int decompid, const PIO_Offset recnum,
                           char *buf);
    int PIOc_get_vard_schar(int ncid, int varid, int decompid, const PIO_Offset recnum,
                            signed char *buf);
    int PIOc_get_vard_short(int ncid, int varid, int decompid, const PIO_Offset recnum,
                            short *buf);
    int PIOc_get_vard_int(int ncid, int varid, int decompid, const PIO_Offset recnum,
                          int *buf);
    int PIOc_get_vard_float(int ncid, int varid, int decompid, const PIO_Offset recnum,
                            float *buf);
    int PIOc_get_vard_double(int ncid, int varid, int decompid, const PIO_Offset recnum,
                             double *buf);
    int PIOc_get_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum,
                            unsigned char *buf);
    int PIOc_get_vard_ushort(int ncid, int varid, int decompid, const PIO_Offset recnum,
                             unsigned short *buf);
    int PIOc_get_vard_uint(int ncid, int varid, int decompid, const PIO_Offset recnum,
                           unsigned int *buf);
    int PIOc_get_vard_longlong(int ncid, int varid, int decompid, const PIO_Offset recnum,
                               long long *buf);
    int PIOc_get_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum,
                                unsigned long long *buf);
 
    /* Data writes - vard. */
    int PIOc_put_vard(int ncid, int varid, int decompid, const PIO_Offset recnum,
                      const void *buf);
    int PIOc_put_vard_text(int ncid, int varid, int decompid, const PIO_Offset recnum,
                           const char *op);
    int PIOc_put_vard_schar(int ncid, int varid, int decompid, const PIO_Offset recnum,
                            const signed char *op);
    int PIOc_put_vard_short(int ncid, int varid, int decompid, const PIO_Offset recnum,
                            const short *op);
    int PIOc_put_vard_int(int ncid, int varid, int decompid, const PIO_Offset recnum,
                          const int *op);
    int PIOc_put_vard_float(int ncid, int varid, int decompid, const PIO_Offset recnum,
                            const float *op);
    int PIOc_put_vard_double(int ncid, int varid, int decompid, const PIO_Offset recnum,
                             const double *op);
    int PIOc_put_vard_uchar(int ncid, int varid, int decompid, const PIO_Offset recnum,
                            const unsigned char *op);
    int PIOc_put_vard_ushort(int ncid, int varid, int decompid, const PIO_Offset recnum,
                             const unsigned short *op);
    int PIOc_put_vard_uint(int ncid, int varid, int decompid, const PIO_Offset recnum,
                           const unsigned int *op);
    int PIOc_put_vard_longlong(int ncid, int varid, int decompid, const PIO_Offset recnum,
                               const long long *op);
    int PIOc_put_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum,
                                const unsigned long long *op);
 
    /* These functions are for the netCDF integration layer. */
    int nc_def_iosystem(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int rearr,
                         int *iosysidp);
 
    int nc_def_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
                     int component_count, int *num_procs_per_comp, int **proc_list,
                     MPI_Comm *io_comm, MPI_Comm *comp_comm, int rearranger,
                     int *iosysidp);
 
    /* Set the default IOsystem ID. */
    int nc_set_iosystem(int iosysid);
 
    /* Get the default IOsystem ID. */
    int nc_get_iosystem(int *iosysid);
 
    /* Release the resources associated with an iosystem. */
    int nc_free_iosystem(int iosysid);
 
    /* Define a decomposition for distributed arrays. */
    int nc_def_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen,
                      int maplen, const size_t *compmap, int *ioidp,
                      int rearranger, const size_t *iostart,
                      const size_t *iocount);
 
    /* Release resources associated with a decomposition. */
    int nc_free_decomp(int ioid);
 
    /* Data reads - read a distributed array. */
    int nc_get_vard(int ncid, int varid, int decompid, const size_t recnum, void *buf);
    int nc_get_vard_text(int ncid, int varid, int decompid, const size_t recnum,
                         char *buf);
    int nc_get_vard_schar(int ncid, int varid, int decompid, const size_t recnum,
                          signed char *buf);
    int nc_get_vard_short(int ncid, int varid, int decompid, const size_t recnum,
                          short *buf);
    int nc_get_vard_int(int ncid, int varid, int decompid, const size_t recnum,
                        int *buf);
    int nc_get_vard_float(int ncid, int varid, int decompid, const size_t recnum,
                          float *buf);
    int nc_get_vard_double(int ncid, int varid, int decompid, const size_t recnum,
                           double *buf);
    int nc_get_vard_uchar(int ncid, int varid, int decompid, const size_t recnum,
                          unsigned char *buf);
    int nc_get_vard_ushort(int ncid, int varid, int decompid, const size_t recnum,
                           unsigned short *buf);
    int nc_get_vard_uint(int ncid, int varid, int decompid, const size_t recnum,
                         unsigned int *buf);
    int nc_get_vard_longlong(int ncid, int varid, int decompid, const size_t recnum,
                             long long *buf);
    int nc_get_vard_ulonglong(int ncid, int varid, int decompid, const size_t recnum,
                              unsigned long long *buf);
 
    /* Data writes - Write a distributed array. */
    int nc_put_vard(int ncid, int varid, int decompid, const size_t recnum,
                    const void *buf);
    int nc_put_vard_text(int ncid, int varid, int decompid, const size_t recnum,
                         const char *op);
    int nc_put_vard_schar(int ncid, int varid, int decompid, const size_t recnum,
                          const signed char *op);
    int nc_put_vard_short(int ncid, int varid, int decompid, const size_t recnum,
                          const short *op);
    int nc_put_vard_int(int ncid, int varid, int decompid, const size_t recnum,
                        const int *op);
    int nc_put_vard_float(int ncid, int varid, int decompid, const size_t recnum,
                          const float *op);
    int nc_put_vard_double(int ncid, int varid, int decompid, const size_t recnum,
                           const double *op);
    int nc_put_vard_uchar(int ncid, int varid, int decompid, const size_t recnum,
                          const unsigned char *op);
    int nc_put_vard_ushort(int ncid, int varid, int decompid, const size_t recnum,
                           const unsigned short *op);
    int nc_put_vard_uint(int ncid, int varid, int decompid, const size_t recnum,
                         const unsigned int *op);
    int nc_put_vard_longlong(int ncid, int varid, int decompid, const size_t recnum,
                             const long long *op);
    int nc_put_vard_ulonglong(int ncid, int varid, int decompid, const size_t recnum,
                              const unsigned long long *op);
 
#if defined(__cplusplus)
}
#endif
 
#endif  // _PIO_H_