Merge flight parameters onto obspack IDs
Merge flight parameters onto obspack IDsΒΆ
R program to merge flight parameters from campaign 10-sec merge files onto obspack IDs for use by process_aircraft_models.ipynb
Notes:
merging is needed because model files generally do not include pressure, theta, campaign, flight, and profile variables
also models either do not report or round time and position values, such that merging on those not possible, so using 2 steps:
merge 10-sec flight data and NOAA Obpack here and write out
in process_aircraft_models.ipynb, merge model output with these output files
because the model output corresponds to many different ObsPack versions, need to download each and match flight data for each
CT2017, CT2019B, CTE2020, MIROC, TM5pCO2, and CAMS have obspack_id variables (CAMS only has obspack_id and co2)
CarboScope text files do not include obspack_id, so need to merge based on row matching
library('ncdf4')
library('yaml')
project_tmpdir_obs = read_yaml('../_config_calc.yml')$project_tmpdir_obs
username = Sys.info()['user']
project_tmpdir_obs = gsub('\\{\\{env\\[\'USER\'\\]\\}\\}', username, project_tmpdir_obs)
# read in preprocessed aircraft files from process_aircraft_10s.ipynb
load('HIPPO_10s.RData')
load('ORCAS_10s.RData')
load('ATom_10s.RData')
# point to ObsPack directories and define short names
ncdirs=c(
paste(project_tmpdir_obs,'/obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05/data/nc',sep=''),
paste(project_tmpdir_obs,'/obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12/data/nc',sep=''), # CT2019B includes this
paste(project_tmpdir_obs,'/obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11/data/nc',sep=''),
paste(project_tmpdir_obs,'/obspack_co2_1_ATom_v4.0_2020-04-06/data/nc',sep=''),
paste(project_tmpdir_obs,'/obspack_co2_1_CARBONTRACKER_CT2017_2018-05-02/data/nc',sep='')
# CT2017 includes these 3: obspack_co2_1_ORCAS_v2.0_2017-04-05, obspack_co2_1_GLOBALVIEWplus_v3.1_2017-10-18, obspack_co2_1_NRT_v4.0_2017-09-08 (includes ATom-2 but fluxes only go through 12/2016 so not included in paper)
)
ops=c('GLOBALVIEWplus_v4.2.2','GLOBALVIEWplus_v5.0','GLOBALVIEWplus_v6.0','ATom_v4.0','CARBONTRACKER_CT2017')
# specify file names and campaigns
ncfiles=c( 'co2_hip_aircraft-insitu_59_allvalid.nc', 'co2_orc_aircraft-insitu_3_allvalid-merge10.nc', 'co2_tom_aircraft-insitu_1_allvalid.nc' )
camps=c('HIPPO','ORCAS','ATOM')
# loop on ObsPack, merge, and write out
for(i in c(1:length(ncdirs))){
ncdir=ncdirs[i]
op=ops[i]
# loop on campaign
for(j in c(1:3)){
camp=camps[j]
ncfile=ncfiles[j]
if(!grepl('ATom',ncdir)|camp=='ATOM'){ # only process ATom v4.0 ObsPack for ATom
print(paste(op,camp))
ncin=nc_open(paste(ncdir,'/',ncfile,sep=''))
ncdat=data.frame(cbind(t(ncvar_get(ncin,'time_components')),ncvar_get(ncin,'obspack_id')),stringsAsFactors=F) ; colnames(ncdat)=c('year','mon','day','hour','min','sec','obspack_id')
print(head(ncdat))
mergefile=get(paste(tolower(camp),'merge',sep='')) # from .RData load above
print(dim(mergefile))
print(dim(ncdat))
mrgdat=merge(mergefile,ncdat,by=c('year','mon','day','hour','min','sec'))
print(dim(mrgdat))
if(camp=='ATOM'&!grepl('CT2017',ncdir)){ # ATom (except for CT2017) fails to match 7925 because ObsPack includes test flights that are not in the 10s merge product. Only report days:
print(unique(paste(ncdat$year,ncdat$mon,ncdat$day)[!is.element(paste(ncdat$year,ncdat$mon,ncdat$day,ncdat$hour,ncdat$min,ncdat$sec),paste(mrgdat$year,mrgdat$mon,mrgdat$day,mrgdat$hour,mrgdat$min,mrgdat$sec))]))
} else { # HIPPO fails to match 6 values because of uneven seconds in the ObsPack, ATom fails to match 11 values in CT2017
print(paste(ncdat$year,ncdat$mon,ncdat$day,ncdat$hour,ncdat$min,ncdat$sec)[!is.element(paste(ncdat$year,ncdat$mon,ncdat$day,ncdat$hour,ncdat$min,ncdat$sec),paste(mrgdat$year,mrgdat$mon,mrgdat$day,mrgdat$hour,mrgdat$min,mrgdat$sec))])
}
if(camp=='ORCAS'){ # no 'camp' variable
mrgdat=mrgdat[,c('year','mon','day','hour','min','sec','flt','prof','pressure','theta','strat','obspack_id')]
write(c('year month day hour min sec flt prof pressure theta strat obspack_id'),paste(camp,'_obspack_',op,'_merge.txt',sep=''))
} else {
mrgdat=mrgdat[,c('year','mon','day','hour','min','sec','camp','flt','prof','pressure','theta','strat','obspack_id')]
write(c('year month day hour min sec camp flt prof pressure theta strat obspack_id'),paste(camp,'_obspack_',op,'_merge.txt',sep=''))
}
write(t(mrgdat),paste(camp,'_obspack_',op,'_merge.txt',sep=''),ncol=ncol(mrgdat),append=T)
}
}
}
[1] "GLOBALVIEWplus_v4.2.2 HIPPO"
year mon day hour min sec
1 2009 1 9 20 28 40
2 2009 1 9 20 28 50
3 2009 1 9 20 29 0
4 2009 1 9 20 29 10
5 2009 1 9 20 29 20
6 2009 1 9 20 29 30
obspack_id
1 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_hip_aircraft-insitu_59_allvalid~14594606
2 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_hip_aircraft-insitu_59_allvalid~14594607
3 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_hip_aircraft-insitu_59_allvalid~14594608
4 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_hip_aircraft-insitu_59_allvalid~14594609
5 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_hip_aircraft-insitu_59_allvalid~14594610
6 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_hip_aircraft-insitu_59_allvalid~14594611
[1] 156551 28
[1] 130016 7
[1] 130010 29
[1] "2009 11 11 21 26 13" "2010 4 16 1 59 57" "2011 8 24 20 0 13"
[4] "2011 8 29 22 16 21" "2011 9 1 21 59 33" "2011 9 9 16 11 23"
[1] "GLOBALVIEWplus_v4.2.2 ORCAS"
year mon day hour min sec
1 2016 1 5 19 19 54
2 2016 1 5 19 20 4
3 2016 1 5 19 25 34
4 2016 1 5 19 25 44
5 2016 1 5 19 25 54
6 2016 1 5 19 26 4
obspack_id
1 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_orc_aircraft-insitu_3_allvalid-merge10~20017854
2 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_orc_aircraft-insitu_3_allvalid-merge10~20017855
3 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_orc_aircraft-insitu_3_allvalid-merge10~20017856
4 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_orc_aircraft-insitu_3_allvalid-merge10~20017857
5 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_orc_aircraft-insitu_3_allvalid-merge10~20017858
6 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_orc_aircraft-insitu_3_allvalid-merge10~20017859
[1] 46646 22
[1] 40217 7
[1] 40217 23
character(0)
[1] "GLOBALVIEWplus_v4.2.2 ATOM"
year mon day hour min sec
1 2016 8 3 19 3 10
2 2016 8 3 19 3 20
3 2016 8 3 19 3 30
4 2016 8 3 19 3 40
5 2016 8 3 19 3 50
6 2016 8 3 19 4 0
obspack_id
1 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_tom_aircraft-insitu_1_allvalid~27984457
2 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_tom_aircraft-insitu_1_allvalid~27984458
3 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_tom_aircraft-insitu_1_allvalid~27984459
4 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_tom_aircraft-insitu_1_allvalid~27984460
5 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_tom_aircraft-insitu_1_allvalid~27984461
6 obspack_co2_1_GLOBALVIEWplus_v4.2.2_2019-06-05~co2_tom_aircraft-insitu_1_allvalid~27984462
[1] 149133 28
[1] 95399 7
[1] 95399 29
character(0)
[1] "GLOBALVIEWplus_v5.0 HIPPO"
year mon day hour min sec
1 2009 1 9 20 28 40
2 2009 1 9 20 28 50
3 2009 1 9 20 29 0
4 2009 1 9 20 29 10
5 2009 1 9 20 29 20
6 2009 1 9 20 29 30
obspack_id
1 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_hip_aircraft-insitu_59_allvalid~6556353
2 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_hip_aircraft-insitu_59_allvalid~6556354
3 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_hip_aircraft-insitu_59_allvalid~6556355
4 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_hip_aircraft-insitu_59_allvalid~6556356
5 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_hip_aircraft-insitu_59_allvalid~6556357
6 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_hip_aircraft-insitu_59_allvalid~6556358
[1] 156551 28
[1] 130016 7
[1] 130010 29
[1] "2009 11 11 21 26 13" "2010 4 16 1 59 57" "2011 8 24 20 0 13"
[4] "2011 8 29 22 16 21" "2011 9 1 21 59 33" "2011 9 9 16 11 23"
[1] "GLOBALVIEWplus_v5.0 ORCAS"
year mon day hour min sec
1 2016 1 5 19 19 54
2 2016 1 5 19 20 4
3 2016 1 5 19 25 34
4 2016 1 5 19 25 44
5 2016 1 5 19 25 54
6 2016 1 5 19 26 4
obspack_id
1 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_orc_aircraft-insitu_3_allvalid-merge10~11693773
2 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_orc_aircraft-insitu_3_allvalid-merge10~11693774
3 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_orc_aircraft-insitu_3_allvalid-merge10~11693775
4 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_orc_aircraft-insitu_3_allvalid-merge10~11693776
5 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_orc_aircraft-insitu_3_allvalid-merge10~11693777
6 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_orc_aircraft-insitu_3_allvalid-merge10~11693778
[1] 46646 22
[1] 40217 7
[1] 40217 23
character(0)
[1] "GLOBALVIEWplus_v5.0 ATOM"
year mon day hour min sec
1 2016 8 3 19 3 10
2 2016 8 3 19 3 20
3 2016 8 3 19 3 30
4 2016 8 3 19 3 40
5 2016 8 3 19 3 50
6 2016 8 3 19 4 0
obspack_id
1 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_tom_aircraft-insitu_1_allvalid~11414194
2 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_tom_aircraft-insitu_1_allvalid~11414195
3 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_tom_aircraft-insitu_1_allvalid~11414196
4 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_tom_aircraft-insitu_1_allvalid~11414197
5 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_tom_aircraft-insitu_1_allvalid~11414198
6 obspack_co2_1_GLOBALVIEWplus_v5.0_2019-08-12~co2_tom_aircraft-insitu_1_allvalid~11414199
[1] 149133 28
[1] 143040 7
[1] 135115 29
[1] "2017 9 14" "2017 9 26" "2018 4 10" "2018 4 11" "2018 4 13" "2018 4 14"
[1] "GLOBALVIEWplus_v6.0 HIPPO"
year mon day hour min sec
1 2009 1 9 20 28 40
2 2009 1 9 20 28 50
3 2009 1 9 20 29 0
4 2009 1 9 20 29 10
5 2009 1 9 20 29 20
6 2009 1 9 20 29 30
obspack_id
1 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_hip_aircraft-insitu_59_allvalid~6799852
2 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_hip_aircraft-insitu_59_allvalid~6799853
3 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_hip_aircraft-insitu_59_allvalid~6799854
4 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_hip_aircraft-insitu_59_allvalid~6799855
5 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_hip_aircraft-insitu_59_allvalid~6799856
6 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_hip_aircraft-insitu_59_allvalid~6799857
[1] 156551 28
[1] 130016 7
[1] 130010 29
[1] "2009 11 11 21 26 13" "2010 4 16 1 59 57" "2011 8 24 20 0 13"
[4] "2011 8 29 22 16 21" "2011 9 1 21 59 33" "2011 9 9 16 11 23"
[1] "GLOBALVIEWplus_v6.0 ORCAS"
year mon day hour min sec
1 2016 1 5 19 19 54
2 2016 1 5 19 20 4
3 2016 1 5 19 25 34
4 2016 1 5 19 25 44
5 2016 1 5 19 25 54
6 2016 1 5 19 26 4
obspack_id
1 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_orc_aircraft-insitu_3_allvalid-merge10~12376263
2 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_orc_aircraft-insitu_3_allvalid-merge10~12376264
3 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_orc_aircraft-insitu_3_allvalid-merge10~12376265
4 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_orc_aircraft-insitu_3_allvalid-merge10~12376266
5 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_orc_aircraft-insitu_3_allvalid-merge10~12376267
6 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_orc_aircraft-insitu_3_allvalid-merge10~12376268
[1] 46646 22
[1] 40217 7
[1] 40217 23
character(0)
[1] "GLOBALVIEWplus_v6.0 ATOM"
year mon day hour min sec
1 2016 7 29 14 55 10
2 2016 7 29 14 55 20
3 2016 7 29 14 55 30
4 2016 7 29 14 55 40
5 2016 7 29 14 55 50
6 2016 7 29 14 56 0
obspack_id
1 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_tom_aircraft-insitu_1_allvalid~12061595
2 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_tom_aircraft-insitu_1_allvalid~12061596
3 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_tom_aircraft-insitu_1_allvalid~12061597
4 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_tom_aircraft-insitu_1_allvalid~12061598
5 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_tom_aircraft-insitu_1_allvalid~12061599
6 obspack_co2_1_GLOBALVIEWplus_v6.0_2020-09-11~co2_tom_aircraft-insitu_1_allvalid~12061600
[1] 149133 28
[1] 147660 7
[1] 139735 29
[1] "2017 9 14" "2017 9 26" "2018 4 10" "2018 4 11" "2018 4 13" "2018 4 14"
[1] "ATom_v4.0 ATOM"
year mon day hour min sec
1 2016 7 29 14 55 10
2 2016 7 29 14 55 20
3 2016 7 29 14 55 30
4 2016 7 29 14 55 40
5 2016 7 29 14 55 50
6 2016 7 29 14 56 0
obspack_id
1 obspack_co2_1_ATom_v4.0_2020-04-06~co2_tom_aircraft-insitu_1_allvalid~1
2 obspack_co2_1_ATom_v4.0_2020-04-06~co2_tom_aircraft-insitu_1_allvalid~2
3 obspack_co2_1_ATom_v4.0_2020-04-06~co2_tom_aircraft-insitu_1_allvalid~3
4 obspack_co2_1_ATom_v4.0_2020-04-06~co2_tom_aircraft-insitu_1_allvalid~4
5 obspack_co2_1_ATom_v4.0_2020-04-06~co2_tom_aircraft-insitu_1_allvalid~5
6 obspack_co2_1_ATom_v4.0_2020-04-06~co2_tom_aircraft-insitu_1_allvalid~6
[1] 149133 28
[1] 147660 7
[1] 139735 29
[1] "2017 9 14" "2017 9 26" "2018 4 10" "2018 4 11" "2018 4 13" "2018 4 14"
[1] "CARBONTRACKER_CT2017 HIPPO"
year mon day hour min sec
1 2009 1 9 20 28 40
2 2009 1 9 20 28 50
3 2009 1 9 20 29 0
4 2009 1 9 20 29 10
5 2009 1 9 20 29 20
6 2009 1 9 20 29 30
obspack_id
1 obspack_co2_1_GLOBALVIEWplus_v3.1_2017-10-18~co2_hip_aircraft-insitu_59_allvalid~4052621------------
2 obspack_co2_1_GLOBALVIEWplus_v3.1_2017-10-18~co2_hip_aircraft-insitu_59_allvalid~4052622------------
3 obspack_co2_1_GLOBALVIEWplus_v3.1_2017-10-18~co2_hip_aircraft-insitu_59_allvalid~4052623------------
4 obspack_co2_1_GLOBALVIEWplus_v3.1_2017-10-18~co2_hip_aircraft-insitu_59_allvalid~4052624------------
5 obspack_co2_1_GLOBALVIEWplus_v3.1_2017-10-18~co2_hip_aircraft-insitu_59_allvalid~4052625------------
6 obspack_co2_1_GLOBALVIEWplus_v3.1_2017-10-18~co2_hip_aircraft-insitu_59_allvalid~4052626------------
[1] 156551 28
[1] 130016 7
[1] 130010 29
[1] "2009 11 11 21 26 13" "2010 4 16 1 59 57" "2011 8 24 20 0 13"
[4] "2011 8 29 22 16 21" "2011 9 1 21 59 33" "2011 9 9 16 11 23"
[1] "CARBONTRACKER_CT2017 ORCAS"
year mon day hour min sec
1 2016 1 5 19 19 54
2 2016 1 5 19 20 4
3 2016 1 5 19 25 34
4 2016 1 5 19 25 44
5 2016 1 5 19 25 54
6 2016 1 5 19 26 4
obspack_id
1 obspack_co2_1_ORCAS_v2.0_2017-04-05~co2_orc_aircraft-insitu_3_allvalid-merge10~1--------------------
2 obspack_co2_1_ORCAS_v2.0_2017-04-05~co2_orc_aircraft-insitu_3_allvalid-merge10~2--------------------
3 obspack_co2_1_ORCAS_v2.0_2017-04-05~co2_orc_aircraft-insitu_3_allvalid-merge10~3--------------------
4 obspack_co2_1_ORCAS_v2.0_2017-04-05~co2_orc_aircraft-insitu_3_allvalid-merge10~4--------------------
5 obspack_co2_1_ORCAS_v2.0_2017-04-05~co2_orc_aircraft-insitu_3_allvalid-merge10~5--------------------
6 obspack_co2_1_ORCAS_v2.0_2017-04-05~co2_orc_aircraft-insitu_3_allvalid-merge10~6--------------------
[1] 46646 22
[1] 40217 7
[1] 40217 23
character(0)
[1] "CARBONTRACKER_CT2017 ATOM"
year mon day hour min sec
1 2016 8 3 19 3 10
2 2016 8 3 19 3 20
3 2016 8 3 19 3 30
4 2016 8 3 19 3 40
5 2016 8 3 19 3 50
6 2016 8 3 19 4 0
obspack_id
1 obspack_co2_1_GLOBALVIEWplus_v3.1_2017-10-18~co2_tom_aircraft-insitu_1_allvalid~11219187------------
2 obspack_co2_1_GLOBALVIEWplus_v3.1_2017-10-18~co2_tom_aircraft-insitu_1_allvalid~11219188------------
3 obspack_co2_1_GLOBALVIEWplus_v3.1_2017-10-18~co2_tom_aircraft-insitu_1_allvalid~11219189------------
4 obspack_co2_1_GLOBALVIEWplus_v3.1_2017-10-18~co2_tom_aircraft-insitu_1_allvalid~11219190------------
5 obspack_co2_1_GLOBALVIEWplus_v3.1_2017-10-18~co2_tom_aircraft-insitu_1_allvalid~11219191------------
6 obspack_co2_1_GLOBALVIEWplus_v3.1_2017-10-18~co2_tom_aircraft-insitu_1_allvalid~11219192------------
[1] 149133 28
[1] 56047 7
[1] 56036 29
[1] "2017 1 27 3 32 10" "2017 1 30 3 21 50" "2017 2 2 4 20 20"
[4] "2017 2 2 4 20 30" "2017 2 2 4 20 40" "2017 2 2 4 20 50"
[7] "2017 2 2 4 21 0" "2017 2 2 4 21 10" "2017 2 2 4 21 20"
[10] "2017 2 2 4 21 30" "2017 2 6 5 7 40"