############################################################################################### # # This module is used to set up a couple config files then run the Framework for a specified unit under NDE. # - Add GMTCO file name to SDR PCF file # - Add SDR PCF and other needed input files to unit CFG file # - Run the framework! # # Run automatically as part of a top-level driver script JRR_PRODUCT_unit.py # But can also be run manually for testing # # HISTORY: # created May 2017, E. Buzan @IMSG # ############################################################################################### from __future__ import print_function import sys import os import glob import re import netCDF4 import subprocess import time import logging import readpcf day_labels = ['night','day','twilight'] def run_unit(working_dir,config,unit_name): #set up logging my_name_py = os.path.basename(__file__) my_name = my_name_py.split('.')[0] log_filename = '{}/{}.log'.format(working_dir,my_name_py) if os.path.exists(log_filename): os.remove(log_filename) lawg = logging.getLogger(my_name) lawg.setLevel(logging.INFO) fh = logging.FileHandler(log_filename) sh = logging.StreamHandler() form = logging.Formatter('%(name)s - %(levelname)s - %(message)s') fh.setFormatter(form) sh.setFormatter(form) lawg.addHandler(fh) lawg.addHandler(sh) lawg.info('Starting {0} at {1}'.format(my_name,time.strftime('%Y-%m-%d %H:%M:%S'))) #giant try/except block so we can log stack trace if something goes wrong try: os.environ['PATH'] = '/opt/apps/ots/wgrib2:' + os.environ['PATH'] #grab row/column dimensions to add to SDR PCF gmtco_filepath = '{0}/{1}'.format(working_dir,config['gmtco']) gmtco_filepath = re.sub('_c\d{20}_','_',gmtco_filepath) gmtco_data = netCDF4.Dataset(gmtco_filepath) num_rows = len(gmtco_data.dimensions['phony_dim_0']) num_cols = len(gmtco_data.dimensions['phony_dim_1']) lawg.info('This granule has {0} rows and {1} columns'.format(num_rows,num_cols)) #grab time of day day_flag = gmtco_data.variables['ModeGran'][0] lawg.info('This is a {0} granule'.format(day_labels[day_flag])) lawg.info('Unit is {0}'.format(unit_name)) if unit_name in ['JRR_PRODUCT_AEROSOL_AODADP','JRR_PRODUCT_CRYOSPHERE_SNOW','JRR_PRODUCT_VLPS_LSA'] and day_flag == 0: lawg.info('{0} does not run for nighttime granules, stopping'.format(unit_name)) return 'night' #grab timestamp to append to the CFG filename match = re.search('d\d{8}\_t\d{7}\_e\d{7}\_b\d{5}',gmtco_filepath) if match: timestamp = match.group(0) else: raise IOError("Something's wrong with the GMTCO filename, I can't get the timestamp!") #grab satellite name satellite = config['SATELLITE'] lawg.info('Satellite name is {0}'.format(satellite)) if satellite == 'NPP': sat_id = '300' ancil_file_m = 'npp_viirs_ancil.Mbands.nc' elif satellite == 'NOAA20': sat_id = '301' ancil_file_m = 'j1_viirs_ancil.Mbands.nc' elif satellite in ['J1','J01']: lawg.info('changing to NOAA20 for the SDF file') satellite = 'NOAA20' sat_id = '301' ancil_file_m = 'j1_viirs_ancil.Mbands.nc' else: raise IOError('Unknown satellite name, must be one of NPP, NOAA20, J1, J01'.format(satellite)) #need L1B? tell SAT_VIIRS to save it if 'keep_l1b_bands' in config.keys(): output_l1b = 'Y' else: output_l1b = 'N' #LSA needs its filtered tiles if 'DIR_FOR_FILTERED_LSA_INPUT' in config.keys(): os.symlink(config['DIR_FOR_FILTERED_LSA_INPUT'],'{0}/Filtered_LSA_Tile'.format(working_dir)) #LST needs modified LSE if 'PCF_LSE_TEMPLATE' in config.keys(): local_lse_filename = '{0}/scf_emiss_daily_{1}.pcf'.format(working_dir,timestamp) with open(local_lse_filename,'w') as out_file, open(config['PCF_LSE_TEMPLATE'],'r') as template_file: lse_file = config['lse_file'] if type(lse_file) == list: lse_file.sort() lse_file = lse_file[-1] # We want the last one, in alphabetical order (because it has the lastest date) for line in template_file.readlines(): line = re.sub('!lse_file!', lse_file, line) out_file.write(line) #edit and copy SDR PCF lawg.info('Copying SDR PCF file') local_sdr_filename = '{0}/npp_viirs_sdr_{1}.pcf'.format(working_dir,timestamp) with open(local_sdr_filename,'w') as out_file, open(config['PCF_TEMPLATE'],'r') as template_file: for line in template_file.readlines(): line = re.sub('!GMTCO!',gmtco_filepath,line) line = re.sub('!ROW!',str(num_rows),line) line = re.sub('!COL!',str(num_cols),line) line = re.sub('!SATELLITE!',satellite,line) line = re.sub('!SAT_ID!',sat_id,line) line = re.sub('!SDR_ANCIL!',ancil_file_m,line) line = re.sub('!OUTPUT_L1B!',output_l1b,line) out_file.write(line) #edit and copy unit CFG lawg.info('Copying unit CFG file') local_cfg_filename = '{0}/{1}_{2}.cfg'.format(working_dir,unit_name,timestamp) with open(local_cfg_filename,'w') as out_file, open(config['CFG_TEMPLATE'],'r') as template_file: for line in template_file.readlines(): matches = re.findall('!(.+?)!',line) for match in matches: fullmatch = '!{}!'.format(match) lawg.info('changing {} in cfg file'.format(match)) if match == 'working_dir': line = re.sub('!working_dir!',working_dir+'/',line) elif match == 'SDR': line = re.sub('!SDR!',local_sdr_filename,line) elif match == 'LSE': line = re.sub('!LSE!',local_lse_filename,line) elif match == 'ROW': line = re.sub('!ROW!',str(num_rows),line) elif match == 'COL': line = re.sub('!COL!',str(num_cols),line) elif match[0:3] in ['SVM','SVI']: #SDR file channel_lower = match.lower() channel_file = '{0}/{1}'.format(working_dir,config[channel_lower]) channel_file = re.sub('_c\d{20}_','_',channel_file) line = re.sub(fullmatch,channel_file,line) elif match[0:3] == 'JRR': #nc output file TODO: Make this work for NDE jrr_file = 'not_found' #LST night will not have AOD file, this will show up in cfg but will be commented out for key in config.keys(): if match in config[key]: jrr_file = config[key] line = re.sub(fullmatch,jrr_file,line) elif match[0:3] == 'sun': #time-of-day specific changes if match == 'sun_{0}'.format(day_labels[day_flag]): line = re.sub(fullmatch,'',line) else: line = re.sub(fullmatch,'#',line) elif match[0:5] == 'satel': #satellite specific changes if match == 'satel_{0}'.format(satellite): line = re.sub(fullmatch,'',line) else: line = re.sub(fullmatch,'#',line) out_file.write(line) #framework time lawg.info("Setup done, let's run that framework!") lawg.info('Sub-sub-log: framework.log') os.chdir(working_dir) time1 = time.time() # #outputs framework to stdout AND the log # framework_status = 0 # print(' '.join(['./pcf_framework.exe',local_cfg_filename.split('/')[-1],'|& tee framework.log'])) # framework_status = subprocess.call(['./pcf_framework.exe',local_cfg_filename.split('/')[-1],'|& tee framework.log']) # if framework_status == 0: # run_time = time.time() - time1 # lawg.info('Framework completed successfully! Runtime: {0:.1f} seconds'.format(run_time)) # else: # lawg.error('Dumping framework.log: \n') # with open('{0}/framework.log'.format(working_dir),'r') as framework_log: # lawg.error(framework_log) # raise RuntimeError('Framework failed with status {0}'.format(framework_status)) #outputs framework to logs, not stdout lawg.info("If you're running this manually, you won't see any framework output on stdout") try: framework_output = subprocess.check_output(['./pcf_framework.exe',local_cfg_filename.split('/')[-1]],stderr=subprocess.STDOUT) with open('{0}/framework.log'.format(working_dir),'w') as framework_log: framework_log.write(framework_output) run_time = time.time() - time1 lawg.info('Framework completed successfully! Runtime: {0:.1f} seconds'.format(run_time)) except subprocess.CalledProcessError as e: with open('{0}/framework.log'.format(working_dir),'w') as framework_log: framework_log.write(e.output) lawg.error('Framework log below...\n\n'+e.output) raise RuntimeError('Framework failed!') # #outputs framework to stdout, not the log # framework_status = 0 # framework_status = subprocess.call(['./pcf_framework.exe',local_cfg_filename.split('/')[-1]]) # if framework_status == 0: # run_time = time.time() - time1 # lawg.info('Framework completed successfully! Runtime: {0:.1f} seconds'.format(run_time)) # else: # raise RuntimeError('Framework failed with status {0}'.format(framework_status)) lawg.info('Finished {0} at {1}'.format(my_name,time.strftime('%Y-%m-%d %H:%M:%S'))) except: #log the error then reraise so main driver script can catch it lawg.error('Error in {0}! Stack trace below...\n\n'.format(my_name),exc_info=True) fh.close() raise if __name__ == '__main__': try: working_dir = sys.argv[1] unit_name = sys.argv[2] except IndexError: print('Syntax: run_unit.py working_dir unit_name') sys.exit(0) pcf_filename = glob.glob('{0}/*py.PCF'.format(working_dir))[0] config = readpcf.readpcf(pcf_filename) run_unit(working_dir,config,unit_name)