#!/usr/bin/python3

# batch_file_download_from_thredds.py

# modified from Dale Robinson, CW WCN
# Apr 14, 2021

# Use Python 3
# Install the threddsclient and ntpath modules

# MUST HARDCODE
#  1) pathname to directory where data files are to be downloaded to
#        --- WITHOUT FINAL "/" ---
#  2) URL to THREDDS server catalog.html listing
#        --- BUT CHANGE THE catalog.html TO catalog.xml ---
#  3) year for which files are to be downloaded, i.e. for ECN to perform
#     annual replacements of its VIIRS NRT data with VIIRS Sci-Qual data
#        --- if the entire VIIRS mission is reprocessed and all
#            files must be updated, then comment out the lines that
#            specify the year

# N.B.: VY & VZ sectors are downloaded separately

# IMPORTANT:
# Feb 15, 2022 - download not working due to certificate error
#                at star.nesdis.noaa.gov
#              - use direct copy from STAR local disk instead
#     /data/socd6/coastwatch/npp/viirs/science/L3/sector/chlora/daily/VY00
#      ...etc... for other params & sectors


import threddsclient
import ntpath
import urllib.request
import os

# HARDCODE 1)
# directory to save nc files
#   WITHOUT FINAL "/"  !!!!!
#DOWNLD_DIR = 'path to directory to save file without final / '
DOWNLD_DIR = '/data/socd5/ecn/data/viirs/sciqual_ts/tmp_vy'
#DOWNLD_DIR = '/data/socd5/ecn/data/viirs/sciqual_ts/tmp_vz'

# HARDCODE 2)
# xml url for files to download
#   CHANGE URL ".html" TO ".xml" !!!!!
sector_xml_path = 'https://www.star.nesdis.noaa.gov/thredds/catalog/chloraVIIRSnppSectorSCIDailyVY00/catalog.xml'
#sector_xml_path = 'https://www.star.nesdis.noaa.gov/thredds/catalog/chloraVIIRSnppSectorSCIDailyVZ00/catalog.xml'
#sector_xml_path = 'https://www.star.nesdis.noaa.gov/thredds/catalog/kd490VIIRSnppSectorSCIDailyVY00/catalog.xml'
#sector_xml_path = 'https://www.star.nesdis.noaa.gov/thredds/catalog/kd490VIIRSnppSectorSCIDailyVZ00/catalog.xml'

# HARDCODE 3)
# specify year to download here
# but if all files are need, then comment out the lines in code below
yr_to_dwnld=2021

# use sector_xml_path to search for the download urls for all files
urls = threddsclient.download_urls(sector_xml_path)

# loop through each url to download
for url in urls:
    # get just the filename from the url
    out_file_name = ntpath.basename(url)

    # parse filename for year (only works for VIIRS Sci-qual 750m sector files
    #                          of the form:  VRSRCW_B2021346_C1_VY00_chlora.nc )
    fileyr=out_file_name[8:12]

    # use 'fileyr' for downloading only the year of choice
    # or comment out this if-statement if dwnld'ing an entire mission's files
    # (may need to adjust position of lines further below if uncommenting 
    #  the if condition)
    if int(fileyr) == yr_to_dwnld:

    # build the full path and file name to save location
       full_path = os.path.join(DOWNLD_DIR, out_file_name)
   
    # download the file
       print('Downloading: ' + out_file_name)
       urllib.request.urlretrieve(url, full_path)

