#!/home/bfiedler/anaconda3/bin/python3
# v0.13 7 September 2012
# usage example: dayxgrab.py 2012070412
#from __future__ import print_function
import sys,os,urllib.request
import gr2npy
from datetime import datetime, timedelta
timecode=sys.argv[1]
sparse=True
dohttp=True
ftpdir="ftp://nomads.ncdc.noaa.gov/GFS/Grid4/"
httpdir="http://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs."
topdir="/home/bfiedler/play/grib/gfs1deg/"
datecode=timecode[0:10]
yeari=timecode[0:4]
mni=timecode[4:6]
dyi=timecode[6:8]
utci=timecode[8:10]
outdir=topdir+datecode+'/'
getdir=ftpdir+yeari+mni+'/'+datecode+'/'
#hetdir=httpdir+timecode+'/' # not working Sept 2019
#hetdir=httpdir+timecode[0:8]+'/'+timecode[8:10]+'/'
hetdir=httpdir+timecode[0:8]+'/'+timecode[8:10]+'/'+'atmos/'  # 6 April 2022
if not sparse:
    ts= ['{0:03d}'.format(x) for x in range(0,100,6)]
    ts+=['{0:03d}'.format(x) for x in range(102,193,6)]
    ts+=['{0:03d}'.format(x) for x in range(204,385,12)]
else:
#    ts=['{0:03d}'.format(x) for x in range(0,181,12)]
    ts=['{0:03d}'.format(x) for x in range(0,193,6)] # for metrprof.xyz
#print ts 
if os.path.exists(outdir):
    print(outdir+' already exists')
    sys.exit(timecode+' already exists')
else:
    os.makedirs(outdir)
    print('make directory', outdir)
###
for ftime in ts:
    filename='gfs_4_'+yeari+mni+dyi+'_'+utci+'00_'+ftime+'.grb2'
    outfile=outdir+filename
    if not dohttp:
        getfile=getdir+filename
    else:
#        if ftime[0]=='0': ftime=ftime[1:] # seems to be unnecessary post 14 Jan 2015
#        hetfile=hetdir+'gfs.t'+utci+'z.pgrb2f'+ftime #bombs after 14 Jan 2015
#        hetfile=hetdir+'gfs.t'+utci+'z.pgrb2.0p50.f'+ftime #experiment
        hetfile=hetdir+'gfs.t'+utci+'z.pgrb2.1p00.f'+ftime #experiment
    if os.path.isfile(outfile):
        if os.path.getsize(outfile)>10000000: 
            print(outfile,' already exists and is > 10 MB')
            continue
    if not dohttp:
        command='ncftpget -C '+getfile+' '+outfile
        print(command)
    else:
        print(hetfile, outfile)  
    try:
        if not dohttp:
            os.system(command)
        else:
            urllib.request.urlretrieve(hetfile, outfile)
            if os.path.getsize(outfile)<1000000:
                os.remove(outfile)  
                print(outfile,' too small, removed')
            else:
                print(" retrieved ",filename)   
    except:
        print("FAIL: ", filename)
if os.listdir(outdir): 
    gr2npy.dir2npy(outdir) # not empty, process it to npy files
else:
    os.rmdir(outdir) # dir was empty, remove it