Skip to content

Commit 9537d77

Browse files
tsutterleytsutterley
authored
refactor: change from os to pathlib (#87)
* refactor: change from `os` to `pathlib` * fix: updated ssl context to fix deprecation error fix: replace pkg_resources with importlib to fix deprecation error * chore: bump version number * refactor: change from `os` to `pathlib` * feat: can use regions to filter sea ice products in cmr queries fix: use CMR queries to find granules in each download test
1 parent 054fe62 commit 9537d77

46 files changed

Lines changed: 1277 additions & 1001 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎doc/source/api_reference/utilities.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@ General Methods
5353

5454
.. autofunction:: icesat2_toolkit.utilities.from_ftp
5555

56+
.. autofunction:: icesat2_toolkit.utilities._create_default_ssl_context
57+
58+
.. autofunction:: icesat2_toolkit.utilities._create_ssl_context_no_verify
59+
60+
.. autofunction:: icesat2_toolkit.utilities._set_ssl_context_options
61+
5662
.. autofunction:: icesat2_toolkit.utilities.check_connection
5763

5864
.. autofunction:: icesat2_toolkit.utilities.http_list

‎icesat2_toolkit/convert.py‎

Lines changed: 29 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
"""
22
convert.py
3-
Written by Tyler Sutterley (12/2022)
3+
Written by Tyler Sutterley (03/2024)
44
Utilities for converting ICESat-2 HDF5 files into different formats
55
66
PYTHON DEPENDENCIES:
@@ -16,11 +16,8 @@
1616
pandas: Python Data Analysis Library
1717
https://pandas.pydata.org/
1818
19-
PROGRAM DEPENDENCIES:
20-
convert_delta_time.py: converts delta times into Julian and year-decimal
21-
time.py: Utilities for calculating time operations
22-
2319
UPDATE HISTORY:
20+
Updated 03/2024: use pathlib to define and operate on paths
2421
Updated 12/2022: place some imports behind try/except statements
2522
Updated 06/2022: place zarr and pandas imports behind try/except statements
2623
Updated 04/2022: updated docstrings to numpy documentation format
@@ -32,13 +29,12 @@
3229
Updated 08/2020: added output in pandas dataframe for ATL06 and ATL08
3330
Written 06/2020
3431
"""
35-
import os
3632
import re
33+
import pathlib
3734
import warnings
3835
import itertools
3936
import posixpath
4037
import numpy as np
41-
from icesat2_toolkit.convert_delta_time import convert_delta_time
4238

4339
# attempt imports
4440
try:
@@ -115,16 +111,14 @@ def HDF5_to_zarr(self, **kwds):
115111
**kwds: dict
116112
keyword arguments for output zarr converter
117113
"""
118-
# split extension from HDF5 file
119-
if isinstance(self.filename, str):
120-
fileBasename,fileExtension=os.path.splitext(self.filename)
121-
else:
122-
fileBasename,fileExtension=os.path.splitext(self.filename.filename)
123114
# output zarr file
124-
zarr_file = os.path.expanduser(f'{fileBasename}.zarr')
115+
if isinstance(self.filename, (str, pathlib.Path)):
116+
zarr_file = pathlib.Path(self.filename).with_suffix('.zarr')
117+
else:
118+
zarr_file = pathlib.Path(self.filename.filename).with_suffix('.zarr')
125119
# copy everything from the HDF5 file to the zarr file
126120
with h5py.File(self.filename, mode='r') as source:
127-
dest = zarr.open_group(zarr_file,mode='w')
121+
dest = zarr.open_group(zarr_file, mode='w')
128122
# value checks on output zarr
129123
if not hasattr(dest, 'create_dataset'):
130124
raise ValueError('dest must be a group, got {!r}'.format(dest))
@@ -142,16 +136,14 @@ def HDF5_to_HDF5(self, **kwds):
142136
**kwds: dict
143137
keyword arguments for output HDF5 converter
144138
"""
145-
# split extension from HDF5 file
146-
if isinstance(self.filename, str):
147-
fileBasename,fileExtension=os.path.splitext(self.filename)
148-
else:
149-
fileBasename,fileExtension=os.path.splitext(self.filename.filename)
150139
# output HDF5 file
151-
hdf5_file = os.path.expanduser(f'{fileBasename}.h5')
140+
if isinstance(self.filename, (str, pathlib.Path)):
141+
hdf5_file = pathlib.Path(self.filename).with_suffix('.h5')
142+
else:
143+
hdf5_file = pathlib.Path(self.filename.filename).with_suffix('.h5')
152144
# copy everything from the HDF5 file
153145
with h5py.File(self.filename,mode='r') as source:
154-
dest = h5py.File(hdf5_file,mode='w')
146+
dest = h5py.File(hdf5_file, mode='w')
155147
# value checks on output HDF5
156148
if not hasattr(dest, 'create_dataset'):
157149
raise ValueError('dest must be a group, got {!r}'.format(dest))
@@ -270,19 +262,17 @@ def HDF5_to_ascii(self, **kwds):
270262
r'(\d{2})(\d{2})(\d{2})_(\d{4})(\d{2})(\d{2})_(\d{3})_(\d{2})(.*?).h5$')
271263
# split extension from HDF5 file
272264
# extract parameters from ICESat2 HDF5 file
273-
if isinstance(self.filename, str):
274-
fileBasename,fileExtension=os.path.splitext(self.filename)
275-
# extract parameters from ICESat2 HDF5 file
276-
SUB,PRD,HEM,YY,MM,DD,HH,MN,SS,TRK,CYCL,GRAN,RL,VERS,AUX = \
277-
rx.findall(os.path.basename(self.filename)).pop()
265+
if isinstance(self.filename, (str, pathlib.Path)):
266+
hdf5_file = pathlib.Path(self.filename)
278267
else:
279-
fileBasename,fileExtension=os.path.splitext(self.filename.filename)
280-
SUB,PRD,HEM,YY,MM,DD,HH,MN,SS,TRK,CYCL,GRAN,RL,VERS,AUX = \
281-
rx.findall(os.path.basename(self.filename.filename)).pop()
268+
hdf5_file = pathlib.Path(self.filename.filename)
269+
# extract parameters from ICESat2 HDF5 file
270+
SUB,PRD,HEM,YY,MM,DD,HH,MN,SS,TRK,CYCL,GRAN,RL,VERS,AUX = \
271+
rx.findall(hdf5_file.name).pop()
282272
# output file suffix for csv or tab-delimited text
283273
delimiter = ',' if self.reformat == 'csv' else '\t'
284274
# copy bare minimum variables from the HDF5 file to the ascii file
285-
source = h5py.File(self.filename,mode='r')
275+
source = h5py.File(self.filename, mode='r')
286276

287277
# find valid beam groups by testing for particular variables
288278
if (PRD == 'ATL06'):
@@ -400,8 +390,9 @@ def HDF5_to_ascii(self, **kwds):
400390
output = np.column_stack([values[v][valid] for v in vnames])
401391

402392
# output ascii file
403-
ascii_file = f'{fileBasename}_{gtx}.{self.reformat}'
404-
fid = open(os.path.expanduser(ascii_file), mode='w', encoding='utf8')
393+
granule = f'{hdf5_file.stem}_{gtx}.{self.reformat}'
394+
ascii_file = hdf5_file.parent.joinpath(granule)
395+
fid = ascii_file.open(mode='w', encoding='utf8')
405396
# print YAML header to top of file
406397
fid.write('{0}:\n'.format('header'))
407398
# global attributes for file
@@ -460,13 +451,13 @@ def HDF5_to_dataframe(self, **kwds):
460451
r'(\d{2})(\d{2})(\d{2})_(\d{4})(\d{2})(\d{2})_(\d{3})_(\d{2})(.*?).h5$')
461452
# split extension from HDF5 file
462453
# extract parameters from ICESat2 HDF5 file
463-
if isinstance(self.filename, str):
464-
# extract parameters from ICESat2 HDF5 file
465-
SUB,PRD,HEM,YY,MM,DD,HH,MN,SS,TRK,CYCL,GRAN,RL,VERS,AUX = \
466-
rx.findall(os.path.basename(self.filename)).pop()
454+
if isinstance(self.filename, (str, pathlib.Path)):
455+
hdf5_file = pathlib.Path(self.filename)
467456
else:
468-
SUB,PRD,HEM,YY,MM,DD,HH,MN,SS,TRK,CYCL,GRAN,RL,VERS,AUX = \
469-
rx.findall(os.path.basename(self.filename.filename)).pop()
457+
hdf5_file = pathlib.Path(self.filename.filename)
458+
# extract parameters from ICESat2 HDF5 file
459+
SUB,PRD,HEM,YY,MM,DD,HH,MN,SS,TRK,CYCL,GRAN,RL,VERS,AUX = \
460+
rx.findall(hdf5_file.name).pop()
470461

471462
# copy bare minimum variables from the HDF5 file to pandas data frame
472463
source = h5py.File(self.filename,mode='r')

‎icesat2_toolkit/io/ATL03.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env python
22
u"""
3-
ATL03.py (11/2023)
3+
ATL03.py (03/2024)
44
Read ICESat-2 ATL03 and ATL09 data files to calculate average segment surfaces
55
ATL03 datasets: Global Geolocated Photons
66
ATL09 datasets: Atmospheric Characteristics
@@ -15,6 +15,7 @@
1515
https://www.h5py.org/
1616
1717
UPDATE HISTORY:
18+
Updated 03/2024: use pathlib to define and operate on paths
1819
Updated 11/2023: drop DIMENSION_LIST, CLASS and NAME attributes
1920
Updated 12/2022: place some imports behind try/except statements
2021
refactor ICESat-2 data product read programs under io
@@ -33,10 +34,10 @@
3334
"""
3435
from __future__ import print_function, division
3536

36-
import os
3737
import io
3838
import re
3939
import logging
40+
import pathlib
4041
import warnings
4142
import numpy as np
4243
import scipy.interpolate
@@ -72,7 +73,8 @@ def read_granule(FILENAME, ATTRIBUTES=False, **kwargs):
7273
if isinstance(FILENAME, io.IOBase):
7374
fileID = h5py.File(FILENAME, 'r')
7475
else:
75-
fileID = h5py.File(os.path.expanduser(FILENAME), 'r')
76+
FILENAME = pathlib.Path(FILENAME).expanduser().absolute()
77+
fileID = h5py.File(FILENAME, 'r')
7678

7779
# Output HDF5 file information
7880
logging.info(fileID.filename)
@@ -298,7 +300,8 @@ def interpolate_ATL09(FILENAME, pfl, dtime, ATTRIBUTES=True, **kwargs):
298300
if isinstance(FILENAME, io.IOBase):
299301
fileID = h5py.File(FILENAME, 'r')
300302
else:
301-
fileID = h5py.File(os.path.expanduser(FILENAME), 'r')
303+
FILENAME = pathlib.Path(FILENAME).expanduser().absolute()
304+
fileID = h5py.File(FILENAME, 'r')
302305

303306
# allocate python dictionaries for ICESat-2 ATL09 variables and attributes
304307
IS2_atl09_mds = {}
@@ -366,7 +369,8 @@ def find_beams(FILENAME, **kwargs):
366369
if isinstance(FILENAME, io.IOBase):
367370
fileID = h5py.File(FILENAME, 'r')
368371
else:
369-
fileID = h5py.File(os.path.expanduser(FILENAME), 'r')
372+
FILENAME = pathlib.Path(FILENAME).expanduser().absolute()
373+
fileID = h5py.File(FILENAME, 'r')
370374
# output list of beams
371375
IS2_atl03_beams = []
372376
# read each input beam within the file
@@ -411,7 +415,8 @@ def read_main(FILENAME, ATTRIBUTES=False, **kwargs):
411415
if isinstance(FILENAME, io.IOBase):
412416
fileID = h5py.File(FILENAME, 'r')
413417
else:
414-
fileID = h5py.File(os.path.expanduser(FILENAME), 'r')
418+
FILENAME = pathlib.Path(FILENAME).expanduser().absolute()
419+
fileID = h5py.File(FILENAME, 'r')
415420

416421
# Output HDF5 file information
417422
logging.info(fileID.filename)
@@ -582,7 +587,8 @@ def read_beam(FILENAME, gtx, ATTRIBUTES=False, **kwargs):
582587
if isinstance(FILENAME, io.IOBase):
583588
fileID = h5py.File(FILENAME, 'r')
584589
else:
585-
fileID = h5py.File(os.path.expanduser(FILENAME), 'r')
590+
FILENAME = pathlib.Path(FILENAME).expanduser().absolute()
591+
fileID = h5py.File(FILENAME, 'r')
586592

587593
# Output HDF5 file information
588594
logging.info(fileID.filename)

‎icesat2_toolkit/io/ATL06.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env python
22
u"""
3-
ATL06.py (11/2023)
3+
ATL06.py (03/2024)
44
Read ICESat-2 ATL06 (Land Ice Along-Track Height Product) data files
55
66
OPTIONS:
@@ -16,6 +16,7 @@
1616
https://www.h5py.org/
1717
1818
UPDATE HISTORY:
19+
Updated 03/2024: use pathlib to define and operate on paths
1920
Updated 11/2023: drop DIMENSION_LIST, CLASS and NAME attributes
2021
Updated 05/2023: extract more ancillary data from ATL06 files
2122
Updated 12/2022: place some imports behind try/except statements
@@ -32,10 +33,10 @@
3233
"""
3334
from __future__ import print_function
3435

35-
import os
3636
import io
3737
import re
3838
import logging
39+
import pathlib
3940
import warnings
4041
import numpy as np
4142

@@ -75,7 +76,8 @@ def read_granule(FILENAME, ATTRIBUTES=False, HISTOGRAM=False,
7576
if isinstance(FILENAME, io.IOBase):
7677
fileID = h5py.File(FILENAME, 'r')
7778
else:
78-
fileID = h5py.File(os.path.expanduser(FILENAME), 'r')
79+
FILENAME = pathlib.Path(FILENAME).expanduser().absolute()
80+
fileID = h5py.File(FILENAME, 'r')
7981

8082
# Output HDF5 file information
8183
logging.info(fileID.filename)
@@ -286,7 +288,8 @@ def find_beams(FILENAME, **kwargs):
286288
if isinstance(FILENAME, io.IOBase):
287289
fileID = h5py.File(FILENAME, 'r')
288290
else:
289-
fileID = h5py.File(os.path.expanduser(FILENAME), 'r')
291+
FILENAME = pathlib.Path(FILENAME).expanduser().absolute()
292+
fileID = h5py.File(FILENAME, 'r')
290293
# output list of beams
291294
IS2_atl06_beams = []
292295
# read each input beam within the file
@@ -340,7 +343,8 @@ def read_beam(FILENAME, gtx, ATTRIBUTES=False, **kwargs):
340343
if isinstance(FILENAME, io.IOBase):
341344
fileID = h5py.File(FILENAME, 'r')
342345
else:
343-
fileID = h5py.File(os.path.expanduser(FILENAME), 'r')
346+
FILENAME = pathlib.Path(FILENAME).expanduser().absolute()
347+
fileID = h5py.File(FILENAME, 'r')
344348

345349
# Output HDF5 file information
346350
logging.info(fileID.filename)

‎icesat2_toolkit/io/ATL07.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env python
22
u"""
3-
ATL07.py (11/2023)
3+
ATL07.py (03/2024)
44
Read ICESat-2 ATL07 (Sea Ice Height) data files
55
66
PYTHON DEPENDENCIES:
@@ -11,6 +11,7 @@
1111
https://www.h5py.org/
1212
1313
UPDATE HISTORY:
14+
Updated 03/2024: use pathlib to define and operate on paths
1415
Updated 11/2023: drop DIMENSION_LIST, CLASS and NAME attributes
1516
Updated 05/2023: extract more ancillary data from ATL07 files
1617
Updated 12/2022: place some imports behind try/except statements
@@ -25,10 +26,10 @@
2526
"""
2627
from __future__ import print_function
2728

28-
import os
2929
import io
3030
import re
3131
import logging
32+
import pathlib
3233
import warnings
3334
import numpy as np
3435

@@ -63,7 +64,8 @@ def read_granule(FILENAME, ATTRIBUTES=False, **kwargs):
6364
if isinstance(FILENAME, io.IOBase):
6465
fileID = h5py.File(FILENAME, 'r')
6566
else:
66-
fileID = h5py.File(os.path.expanduser(FILENAME), 'r')
67+
FILENAME = pathlib.Path(FILENAME).expanduser().absolute()
68+
fileID = h5py.File(FILENAME, 'r')
6769

6870
# Output HDF5 file information
6971
logging.info(fileID.filename)
@@ -229,7 +231,8 @@ def find_beams(FILENAME, **kwargs):
229231
if isinstance(FILENAME, io.IOBase):
230232
fileID = h5py.File(FILENAME, 'r')
231233
else:
232-
fileID = h5py.File(os.path.expanduser(FILENAME), 'r')
234+
FILENAME = pathlib.Path(FILENAME).expanduser().absolute()
235+
fileID = h5py.File(FILENAME, 'r')
233236
# output list of beams
234237
IS2_atl07_beams = []
235238
# read each input beam within the file
@@ -278,7 +281,8 @@ def read_beam(FILENAME, gtx, ATTRIBUTES=False, **kwargs):
278281
if isinstance(FILENAME, io.IOBase):
279282
fileID = h5py.File(FILENAME, 'r')
280283
else:
281-
fileID = h5py.File(os.path.expanduser(FILENAME), 'r')
284+
FILENAME = pathlib.Path(FILENAME).expanduser().absolute()
285+
fileID = h5py.File(FILENAME, 'r')
282286

283287
# Output HDF5 file information
284288
logging.info(fileID.filename)

0 commit comments

Comments
 (0)