import warnings warnings.filterwarnings("once", category=DeprecationWarning) # noqa: E402 import unittest import os import shutil import time import hashlib import tarfile import pytest import itertools from unittest import mock import numpy as np import pandas as pd from numpy.testing import assert_array_equal, assert_allclose salem = pytest.importorskip('salem') gpd = pytest.importorskip('geopandas') import oggm from oggm import utils, workflow, tasks from oggm.utils import _downloads from oggm import cfg from oggm.tests.funcs import get_test_dir, init_hef, TempEnvironmentVariable from oggm.utils import shape_factor_adhikari from oggm.exceptions import (InvalidParamsError, InvalidDEMError, DownloadVerificationFailedException) pytestmark = pytest.mark.test_env("utils") TEST_GDIR_URL = ('https://cluster.klima.uni-bremen.de/~oggm/' 'test_gdirs/oggm_v1.1/') def clean_dir(testdir): shutil.rmtree(testdir) os.makedirs(testdir) class TestFuncs(unittest.TestCase): def setUp(self): pass def tearDown(self): pass def test_show_versions(self): # just see that it runs out = utils.show_versions() assert len(out) > 30 def test_signchange(self): ts = pd.Series([-2., -1., 1., 2., 3], index=np.arange(5)) sc = utils.signchange(ts) assert_array_equal(sc, [0, 0, 1, 0, 0]) ts = pd.Series([-2., -1., 1., 2., 3][::-1], index=np.arange(5)) sc = utils.signchange(ts) assert_array_equal(sc, [0, 0, 0, 1, 0]) def test_smooth(self): a = np.array([1., 4, 7, 7, 4, 1]) b = utils.smooth1d(a, 3, kernel='mean') assert_allclose(b, [3, 4, 6, 6, 4, 3]) kernel = [0.60653066, 1., 0.60653066] b = utils.smooth1d(a, 3, kernel=kernel) c = utils.smooth1d(a, 3) assert_allclose(b, c) def test_filter_rgi_name(self): name = 'Tustumena Glacier \x9c' expected = 'Tustumena Glacier' self.assertTrue(utils.filter_rgi_name(name), expected) name = 'Hintereisferner À' expected = 'Hintereisferner' self.assertTrue(utils.filter_rgi_name(name), expected) name = 'SPECIAL GLACIER 3' expected = 'Special Glacier' self.assertTrue(utils.filter_rgi_name(name), expected) def test_floatyear_to_date(self): r = utils.floatyear_to_date(0) self.assertEqual(r, (0, 1)) y, m = utils.floatyear_to_date([0, 1]) np.testing.assert_array_equal(y, [0, 1]) np.testing.assert_array_equal(m, [1, 1]) y, m = utils.floatyear_to_date([0.00001, 1.00001]) np.testing.assert_array_equal(y, [0, 1]) np.testing.assert_array_equal(m, [1, 1]) y, m = utils.floatyear_to_date([0.99999, 1.99999]) np.testing.assert_array_equal(y, [0, 1]) np.testing.assert_array_equal(m, [12, 12]) yr = 1998 + 2 / 12 r = utils.floatyear_to_date(yr) self.assertEqual(r, (1998, 3)) yr = 1998 + 1 / 12 r = utils.floatyear_to_date(yr) self.assertEqual(r, (1998, 2)) def test_date_to_floatyear(self): r = utils.date_to_floatyear(0, 1) self.assertEqual(r, 0) r = utils.date_to_floatyear(1, 1) self.assertEqual(r, 1) r = utils.date_to_floatyear([0, 1], [1, 1]) np.testing.assert_array_equal(r, [0, 1]) yr = utils.date_to_floatyear([1998, 1998], [6, 7]) y, m = utils.floatyear_to_date(yr) np.testing.assert_array_equal(y, [1998, 1998]) np.testing.assert_array_equal(m, [6, 7]) yr = utils.date_to_floatyear([1998, 1998], [2, 3]) y, m = utils.floatyear_to_date(yr) np.testing.assert_array_equal(y, [1998, 1998]) np.testing.assert_array_equal(m, [2, 3]) time = pd.date_range('1/1/1800', periods=300*12-11, freq='MS') yr = utils.date_to_floatyear(time.year, time.month) y, m = utils.floatyear_to_date(yr) np.testing.assert_array_equal(y, time.year) np.testing.assert_array_equal(m, time.month) myr = utils.monthly_timeseries(1800, 2099) y, m = utils.floatyear_to_date(myr) np.testing.assert_array_equal(y, time.year) np.testing.assert_array_equal(m, time.month) myr = utils.monthly_timeseries(1800, ny=300) y, m = utils.floatyear_to_date(myr) np.testing.assert_array_equal(y, time.year) np.testing.assert_array_equal(m, time.month) time = pd.period_range('0001-01', '6000-1', freq='M') myr = utils.monthly_timeseries(1, 6000) y, m = utils.floatyear_to_date(myr) np.testing.assert_array_equal(y, time.year) np.testing.assert_array_equal(m, time.month) time = pd.period_range('0001-01', '6000-12', freq='M') myr = utils.monthly_timeseries(1, 6000, include_last_year=True) y, m = utils.floatyear_to_date(myr) np.testing.assert_array_equal(y, time.year) np.testing.assert_array_equal(m, time.month) with self.assertRaises(ValueError): utils.monthly_timeseries(1) def test_hydro_convertion(self): # October y, m = utils.hydrodate_to_calendardate(1, 1, start_month=10) assert (y, m) == (0, 10) y, m = utils.hydrodate_to_calendardate(1, 4, start_month=10) assert (y, m) == (1, 1) y, m = utils.hydrodate_to_calendardate(1, 12, start_month=10) assert (y, m) == (1, 9) y, m = utils.hydrodate_to_calendardate([1, 1, 1], [1, 4, 12], start_month=10) np.testing.assert_array_equal(y, [0, 1, 1]) np.testing.assert_array_equal(m, [10, 1, 9]) y, m = utils.calendardate_to_hydrodate(1, 1, start_month=10) assert (y, m) == (1, 4) y, m = utils.calendardate_to_hydrodate(1, 9, start_month=10) assert (y, m) == (1, 12) y, m = utils.calendardate_to_hydrodate(1, 10, start_month=10) assert (y, m) == (2, 1) y, m = utils.calendardate_to_hydrodate([1, 1, 1], [1, 9, 10], start_month=10) np.testing.assert_array_equal(y, [1, 1, 2]) np.testing.assert_array_equal(m, [4, 12, 1]) # Roundtrip time = pd.period_range('0001-01', '1000-12', freq='M') y, m = utils.calendardate_to_hydrodate(time.year, time.month, start_month=10) y, m = utils.hydrodate_to_calendardate(y, m, start_month=10) np.testing.assert_array_equal(y, time.year) np.testing.assert_array_equal(m, time.month) # April y, m = utils.hydrodate_to_calendardate(1, 1, start_month=4) assert (y, m) == (0, 4) y, m = utils.hydrodate_to_calendardate(1, 4, start_month=4) assert (y, m) == (0, 7) y, m = utils.hydrodate_to_calendardate(1, 9, start_month=4) assert (y, m) == (0, 12) y, m = utils.hydrodate_to_calendardate(1, 10, start_month=4) assert (y, m) == (1, 1) y, m = utils.hydrodate_to_calendardate(1, 12, start_month=4) assert (y, m) == (1, 3) y, m = utils.hydrodate_to_calendardate([1, 1, 1], [1, 4, 12], start_month=4) np.testing.assert_array_equal(y, [0, 0, 1]) np.testing.assert_array_equal(m, [4, 7, 3]) # Roundtrip time = pd.period_range('0001-01', '1000-12', freq='M') y, m = utils.calendardate_to_hydrodate(time.year, time.month, start_month=4) y, m = utils.hydrodate_to_calendardate(y, m, start_month=4) np.testing.assert_array_equal(y, time.year) np.testing.assert_array_equal(m, time.month) def test_rgi_meta(self): cfg.initialize() reg_names, subreg_names = utils.parse_rgi_meta(version='6') assert len(reg_names) == 20 assert reg_names.loc[3].values[0] == 'Arctic Canada North' def test_adhikari_shape_factors(self): factors_rectangular = np.array([0.2, 0.313, 0.558, 0.790, 0.884, 0.929, 0.954, 0.990, 1.]) factors_parabolic = np.array([0.2, 0.251, 0.448, 0.653, 0.748, 0.803, 0.839, 0.917, 1.]) w = np.array([0.1, 0.5, 1, 2, 3, 4, 5, 10, 50]) h = 0.5 * np.ones(w.shape) is_rect = h.copy() np.testing.assert_equal(shape_factor_adhikari(w, h, is_rect), factors_rectangular) np.testing.assert_equal(shape_factor_adhikari(w, h, is_rect*0.), factors_parabolic) def test_clips(self): a = np.arange(50) - 5 assert_array_equal(np.clip(a, None, 10), utils.clip_max(a, 10)) assert_array_equal(np.clip(a, 0, None), utils.clip_min(a, 0)) assert_array_equal(np.clip(a, 0, 10), utils.clip_array(a, 0, 10)) for a in [-1, 2, 12]: assert_array_equal(np.clip(a, 0, 10), utils.clip_scalar(a, 0, 10)) assert_array_equal(np.clip(a, None, 10), utils.clip_max(a, 10)) assert_array_equal(np.clip(a, 0, None), utils.clip_min(a, 0)) def test_clips_performance(self): import timeit n = int(1e5) # Scalar s1 = 'import numpy as np' s2 = 'from oggm import utils' t1 = timeit.timeit('np.clip(1, 0, 10)', number=n, setup=s1) t2 = timeit.timeit('utils.clip_scalar(1, 0, 10)', number=n, setup=s2) assert t2 < t1 t1 = timeit.timeit('np.clip(12, None, 10)', number=n, setup=s1) t2 = timeit.timeit('utils.clip_max(12, 10)', number=n, setup=s2) assert t2 < t1 t1 = timeit.timeit('np.clip(12, 15, None)', number=n, setup=s1) t2 = timeit.timeit('utils.clip_min(12, 15)', number=n, setup=s2) assert t2 < t1 # Array s1 = 'import numpy as np; a = np.arange(50) - 5' s2 = 'from oggm import utils; import numpy as np; a = np.arange(50)-5' t1 = timeit.timeit('np.clip(a, 0, 10)', number=n, setup=s1) t2 = timeit.timeit('utils.clip_array(a, 0, 10)', number=n, setup=s2) # This usually fails as advertised by numpy # (although with np 1.17 not) # assert t2 < t1 t1 = timeit.timeit('np.clip(a, None, 10)', number=n, setup=s1) t2 = timeit.timeit('utils.clip_max(a, 10)', number=n, setup=s2) assert t2 < t1 t1 = timeit.timeit('np.clip(a, 15, None)', number=n, setup=s1) t2 = timeit.timeit('utils.clip_min(a, 15)', number=n, setup=s2) assert t2 < t1 class TestInitialize(unittest.TestCase): def setUp(self): cfg.initialize() self.homedir = os.path.expanduser('~') def test_env_var(self): with TempEnvironmentVariable(OGGM_USE_MULTIPROCESSING='1', OGGM_USE_MP_SPAWN=None): cfg.initialize() assert cfg.PARAMS['use_multiprocessing'] assert cfg.PARAMS['mp_processes'] >= 1 assert not cfg.PARAMS['use_mp_spawn'] with TempEnvironmentVariable(OGGM_USE_MULTIPROCESSING='1', OGGM_USE_MP_SPAWN='1', SLURM_JOB_CPUS_PER_NODE='13'): cfg.initialize() assert cfg.PARAMS['use_multiprocessing'] assert cfg.PARAMS['mp_processes'] == 13 assert cfg.PARAMS['use_mp_spawn'] with TempEnvironmentVariable(OGGM_USE_MULTIPROCESSING='0'): cfg.initialize() assert not cfg.PARAMS['use_multiprocessing'] assert cfg.PARAMS['mp_processes'] >= 1 def test_defaults(self): self.assertFalse(cfg.PATHS['working_dir']) def test_pathsetter(self): cfg.PATHS['working_dir'] = os.path.join('~', 'my_OGGM_wd') expected = os.path.join(self.homedir, 'my_OGGM_wd') self.assertEqual(cfg.PATHS['working_dir'], expected) class TestWorkflowTools(unittest.TestCase): def setUp(self): cfg.initialize() self.testdir = os.path.join(get_test_dir(), 'tmp_gir') self.reset_dir() def tearDown(self): if os.path.exists(self.testdir): shutil.rmtree(self.testdir) def reset_dir(self): utils.mkdir(self.testdir, reset=True) def test_leclercq_data(self): hef_file = utils.get_demo_file('Hintereisferner_RGI5.shp') entity = gpd.read_file(hef_file).iloc[0] cfg.PARAMS['use_intersects'] = False gdir = oggm.GlacierDirectory(entity, base_dir=self.testdir) df = gdir.get_ref_length_data() assert df.name == 'Hintereis' assert len(df) == 105 def test_glacier_characs(self): gdir = init_hef() df = utils.compile_glacier_statistics([gdir], path=False) assert len(df) == 1 df = df.iloc[0] np.testing.assert_allclose(df['dem_mean_elev'], df['flowline_mean_elev'], atol=5) df = utils.compile_climate_statistics([gdir], path=False, add_climate_period=1985) np.testing.assert_allclose(df['tstar_avg_prcp'], 2853, atol=5) np.testing.assert_allclose(df['tstar_avg_prcpsol_max_elev'], 2811, atol=5) np.testing.assert_allclose(df['1970-2000_avg_prcpsol_max_elev'], 2811, atol=200) def test_demo_glacier_id(self): cfg.initialize() assert utils.demo_glacier_id('hef') == 'RGI60-11.00897' assert utils.demo_glacier_id('HeF') == 'RGI60-11.00897' assert utils.demo_glacier_id('HintereisFerner') == 'RGI60-11.00897' assert utils.demo_glacier_id('Mer de Glace') == 'RGI60-11.03643' assert utils.demo_glacier_id('RGI60-11.03643') == 'RGI60-11.03643' assert utils.demo_glacier_id('Mer Glace') is None class TestStartFromTar(unittest.TestCase): def setUp(self): # test directory self.testdir = os.path.join(get_test_dir(), 'tmp_tar_tools') # Init cfg.initialize() cfg.set_intersects_db(utils.get_demo_file('rgi_intersect_oetztal.shp')) # Read in the RGI file rgi_file = utils.get_demo_file('rgi_oetztal.shp') self.rgidf = gpd.read_file(rgi_file) cfg.PARAMS['use_multiprocessing'] = False cfg.PATHS['dem_file'] = utils.get_demo_file('srtm_oetztal.tif') cfg.PATHS['working_dir'] = self.testdir self.clean_dir() def tearDown(self): self.rm_dir() def rm_dir(self): shutil.rmtree(self.testdir) def clean_dir(self): utils.mkdir(self.testdir, reset=True) def test_to_and_from_tar(self): # Go - initialize working directories gdirs = workflow.init_glacier_directories(self.rgidf) workflow.execute_entity_task(tasks.define_glacier_region, gdirs) # End - compress all workflow.execute_entity_task(utils.gdir_to_tar, gdirs) # Test - reopen form tar gdirs = workflow.init_glacier_directories(self.rgidf, from_tar=True, delete_tar=True) for gdir in gdirs: assert gdir.has_file('dem') assert not os.path.exists(gdir.dir + '.tar.gz') workflow.execute_entity_task(tasks.glacier_masks, gdirs) workflow.execute_entity_task(utils.gdir_to_tar, gdirs) gdirs = workflow.init_glacier_directories(self.rgidf, from_tar=True) for gdir in gdirs: assert gdir.has_file('gridded_data') assert os.path.exists(gdir.dir + '.tar.gz') def test_to_and_from_basedir_tar(self): # Go - initialize working directories gdirs = workflow.init_glacier_directories(self.rgidf) workflow.execute_entity_task(tasks.define_glacier_region, gdirs) # End - compress all workflow.execute_entity_task(utils.gdir_to_tar, gdirs) utils.base_dir_to_tar() # Test - reopen form tar gdirs = workflow.init_glacier_directories(self.rgidf, from_tar=True) for gdir in gdirs: assert gdir.has_file('dem') assert not os.path.exists(gdir.dir + '.tar.gz') workflow.execute_entity_task(tasks.glacier_masks, gdirs) workflow.execute_entity_task(utils.gdir_to_tar, gdirs) gdirs = workflow.init_glacier_directories(self.rgidf, from_tar=True) for gdir in gdirs: assert gdir.has_file('gridded_data') assert os.path.exists(gdir.dir + '.tar.gz') def test_to_and_from_tar_new_dir(self): # Go - initialize working directories gdirs = workflow.init_glacier_directories(self.rgidf) workflow.execute_entity_task(tasks.define_glacier_region, gdirs) # End - compress all base_dir = os.path.join(self.testdir, 'new_base_dir') paths = workflow.execute_entity_task(utils.gdir_to_tar, gdirs, base_dir=base_dir) # Test - reopen form tar after copy for p, gdir in zip(paths, gdirs): assert base_dir in p shutil.copyfile(p, os.path.normpath(gdir.dir) + '.tar.gz') gdirs = workflow.init_glacier_directories(self.rgidf, from_tar=True, delete_tar=True) for gdir in gdirs: assert gdir.has_file('dem') assert not os.path.exists(gdir.dir + '.tar.gz') assert gdir.rgi_area_km2 > 0 workflow.execute_entity_task(tasks.glacier_masks, gdirs) def test_to_and_from_tar_string(self): # Go - initialize working directories gdirs = workflow.init_glacier_directories(self.rgidf) workflow.execute_entity_task(tasks.define_glacier_region, gdirs) # End - compress all base_dir = os.path.join(self.testdir, 'new_base_dir') utils.mkdir(base_dir, reset=True) paths = workflow.execute_entity_task(utils.gdir_to_tar, gdirs, base_dir=base_dir, delete=False) # Test - reopen form tar after copy new_base_dir = os.path.join(self.testdir, 'newer_base_dir') utils.mkdir(new_base_dir, reset=True) for p, gdir in zip(paths, gdirs): assert base_dir in p new_gdir = utils.GlacierDirectory(gdir.rgi_id, base_dir=new_base_dir, from_tar=p) assert new_gdir.rgi_area_km2 == gdir.rgi_area_km2 assert new_base_dir in new_gdir.base_dir class TestStartFromOnlinePrepro(unittest.TestCase): def setUp(self): # test directory self.testdir = os.path.join(get_test_dir(), 'tmp_prepro_tools') self.dldir = os.path.join(get_test_dir(), 'dl_cache') # Init cfg.initialize() cfg.PATHS['dl_cache_dir'] = self.dldir # Read in the RGI file rgi_file = utils.get_demo_file('rgi_oetztal.shp') self.rgidf = gpd.read_file(rgi_file) self.rgidf['RGIId'] = [rid.replace('RGI50', 'RGI60') for rid in self.rgidf.RGIId] cfg.PARAMS['use_multiprocessing'] = False cfg.PATHS['working_dir'] = self.testdir self.clean_dir() def tearDown(self): self.rm_dir() def rm_dir(self): shutil.rmtree(self.testdir) shutil.rmtree(self.dldir) def clean_dir(self): utils.mkdir(self.testdir, reset=True) utils.mkdir(self.dldir, reset=True) @mock.patch('oggm.utils._downloads.GDIR_URL', TEST_GDIR_URL) def test_start_from_level_1(self): # Go - initialize working directories gdirs = workflow.init_glacier_directories(self.rgidf.iloc[:2], from_prepro_level=1, prepro_rgi_version='61', prepro_border=20, use_demo_glaciers=False) n_intersects = 0 for gdir in gdirs: assert gdir.has_file('dem') n_intersects += gdir.has_file('intersects') assert n_intersects > 0 workflow.execute_entity_task(tasks.glacier_masks, gdirs) @mock.patch('oggm.utils._downloads.GDIR_URL', TEST_GDIR_URL) def test_start_from_level_1_str(self): # Go - initialize working directories entitites = self.rgidf.iloc[:2].RGIId cfg.PARAMS['border'] = 20 gdirs = workflow.init_glacier_directories(entitites, from_prepro_level=1, use_demo_glaciers=False) n_intersects = 0 for gdir in gdirs: assert gdir.has_file('dem') n_intersects += gdir.has_file('intersects') assert n_intersects > 0 workflow.execute_entity_task(tasks.glacier_masks, gdirs) # One string cfg.PARAMS['border'] = 20 gdirs = workflow.init_glacier_directories('RGI60-11.00897', from_prepro_level=1, use_demo_glaciers=False) n_intersects = 0 for gdir in gdirs: assert gdir.has_file('dem') n_intersects += gdir.has_file('intersects') assert n_intersects > 0 workflow.execute_entity_task(tasks.glacier_masks, gdirs) @mock.patch('oggm.utils._downloads.GDIR_URL', TEST_GDIR_URL) def test_start_from_level_2(self): # Go - initialize working directories gdirs = workflow.init_glacier_directories(self.rgidf.iloc[:2], from_prepro_level=2, prepro_rgi_version='61', prepro_border=20, use_demo_glaciers=False) n_intersects = 0 for gdir in gdirs: assert gdir.has_file('dem') assert gdir.has_file('climate_historical') n_intersects += gdir.has_file('intersects') assert n_intersects > 0 workflow.execute_entity_task(tasks.glacier_masks, gdirs) @mock.patch('oggm.utils._downloads.GDIR_URL', TEST_GDIR_URL) def test_start_from_level_3(self): # Go - initialize working directories gdirs = workflow.init_glacier_directories(self.rgidf.iloc[:2], from_prepro_level=3, prepro_rgi_version='61', prepro_border=20, use_demo_glaciers=False) n_intersects = 0 for gdir in gdirs: assert gdir.has_file('dem') assert gdir.has_file('gridded_data') assert gdir.has_file('climate_historical') assert gdir.has_file('climate_info') n_intersects += gdir.has_file('intersects') assert n_intersects > 0 assert gdir.get_climate_info() assert gdir.read_pickle('climate_info') fls = gdir.read_pickle('inversion_flowlines') with pytest.raises(AttributeError): # This is going to raise as long as we keep the "old" prepro gdirs # remove this test after update. # At least we can read the old pickles fls[0].widths_m df = utils.compile_glacier_statistics(gdirs) assert 'dem_med_elev' in df df = utils.compile_climate_statistics(gdirs, add_climate_period=[1920, 1960, 2000]) assert 'tstar_avg_temp_mean_elev' in df assert '1905-1935_avg_temp_mean_elev' in df workflow.execute_entity_task(tasks.init_present_time_glacier, gdirs) @mock.patch('oggm.utils._downloads.GDIR_URL', TEST_GDIR_URL) def test_start_from_level_4(self): # Go - initialize working directories gdirs = workflow.init_glacier_directories(self.rgidf.iloc[:2], from_prepro_level=4, prepro_rgi_version='61', prepro_border=20, use_demo_glaciers=False) workflow.execute_entity_task(tasks.run_random_climate, gdirs, nyears=10) def test_start_from_demo(self): # Go - initialize working directories gdirs = workflow.init_glacier_directories(['kwf', 'hef'], from_prepro_level=4, prepro_rgi_version='61', prepro_border=10) workflow.execute_entity_task(tasks.run_random_climate, gdirs, nyears=10) def test_corrupted_file(self): # Go - initialize working directories gdirs = workflow.init_glacier_directories(['hef'], from_prepro_level=4, prepro_rgi_version='61', prepro_border=10) cfile = utils.get_prepro_gdir('61', 'RGI60-11.00787', 10, 4, base_url=utils.DEMO_GDIR_URL) assert 'cluster.klima.uni-bremen.de/~oggm/' in cfile # Replace with a dummy file os.remove(cfile) with open(cfile, 'w') as f: f.write('ups') # Since we already verified this will error with pytest.raises(tarfile.ReadError): gdirs = workflow.init_glacier_directories(['hef'], from_prepro_level=4, prepro_rgi_version='61', prepro_border=10) # This should retrigger a download and just work cfg.DL_VERIFIED.clear() gdirs = workflow.init_glacier_directories(['hef'], from_prepro_level=4, prepro_rgi_version='61', prepro_border=10) workflow.execute_entity_task(tasks.run_random_climate, gdirs, nyears=10) class TestPreproCLI(unittest.TestCase): def setUp(self): from _pytest.monkeypatch import MonkeyPatch self.monkeypatch = MonkeyPatch() self.testdir = os.path.join(get_test_dir(), 'tmp_prepro_levs') self.reset_dir() def tearDown(self): if os.path.exists(self.testdir): shutil.rmtree(self.testdir) def reset_dir(self): utils.mkdir(self.testdir, reset=True) def test_parse_args(self): from oggm.cli import prepro_levels kwargs = prepro_levels.parse_args(['--rgi-reg', '1', '--map-border', '160']) assert 'working_dir' in kwargs assert 'output_folder' in kwargs assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '01' assert kwargs['border'] == 160 with pytest.raises(InvalidParamsError): prepro_levels.parse_args([]) with pytest.raises(InvalidParamsError): prepro_levels.parse_args(['--rgi-reg', '1']) with pytest.raises(InvalidParamsError): prepro_levels.parse_args(['--map-border', '160']) with TempEnvironmentVariable(OGGM_RGI_REG='1', OGGM_MAP_BORDER='160'): kwargs = prepro_levels.parse_args([]) assert 'working_dir' in kwargs assert 'output_folder' in kwargs assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '01' assert kwargs['border'] == 160 assert not kwargs['is_test'] with pytest.raises(InvalidParamsError): prepro_levels.parse_args([]) kwargs = prepro_levels.parse_args(['--demo', '--map-border', '160', '--output', 'local/out', '--working-dir', 'local/work', ]) assert 'working_dir' in kwargs assert 'output_folder' in kwargs assert 'local' in kwargs['working_dir'] assert 'local' in kwargs['output_folder'] assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '00' assert kwargs['dem_source'] == '' assert kwargs['border'] == 160 assert not kwargs['is_test'] assert kwargs['demo'] assert not kwargs['disable_mp'] assert kwargs['max_level'] == 4 kwargs = prepro_levels.parse_args(['--rgi-reg', '1', '--map-border', '160', '--output', 'local/out', '--working-dir', 'local/work', '--dem-source', 'ALL', ]) assert 'working_dir' in kwargs assert 'output_folder' in kwargs assert 'local' in kwargs['working_dir'] assert 'local' in kwargs['output_folder'] assert kwargs['dem_source'] == 'ALL' assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '01' assert kwargs['border'] == 160 assert not kwargs['is_test'] kwargs = prepro_levels.parse_args(['--rgi-reg', '1', '--map-border', '160', '--output', 'local/out', '--working-dir', 'local/work', '--test', ]) assert 'local' in kwargs['working_dir'] assert 'local' in kwargs['output_folder'] assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '01' assert kwargs['border'] == 160 assert kwargs['is_test'] kwargs = prepro_levels.parse_args(['--rgi-reg', '1', '--map-border', '160', '--output', '/local/out', '--working-dir', '/local/work', ]) assert 'tests' not in kwargs['working_dir'] assert 'tests' not in kwargs['output_folder'] assert 'local' in kwargs['working_dir'] assert 'local' in kwargs['output_folder'] assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '01' assert kwargs['border'] == 160 with TempEnvironmentVariable(OGGM_RGI_REG='12', OGGM_MAP_BORDER='120', OGGM_OUTDIR='local/out', OGGM_WORKDIR='local/work', ): kwargs = prepro_levels.parse_args([]) assert 'local' in kwargs['working_dir'] assert 'local' in kwargs['output_folder'] assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '12' assert kwargs['border'] == 120 with TempEnvironmentVariable(OGGM_RGI_REG='12', OGGM_MAP_BORDER='120', OGGM_OUTDIR='/local/out', OGGM_WORKDIR='/local/work', ): kwargs = prepro_levels.parse_args([]) assert 'local' in kwargs['working_dir'] assert 'local' in kwargs['output_folder'] assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '12' assert kwargs['border'] == 120 def test_full_run(self): from oggm.cli.prepro_levels import run_prepro_levels # Read in the RGI file inter = gpd.read_file(utils.get_demo_file('rgi_intersect_oetztal.shp')) rgidf = gpd.read_file(utils.get_demo_file('rgi_oetztal.shp')) rgidf['RGIId'] = [rid.replace('RGI50', 'RGI60') for rid in rgidf.RGIId] inter['RGIId_1'] = [rid.replace('RGI50', 'RGI60') for rid in inter.RGIId_1] inter['RGIId_2'] = [rid.replace('RGI50', 'RGI60') for rid in inter.RGIId_2] cru_file = utils.get_demo_file('cru_ts3.23.1901.2014.tmp.dat.nc') wdir = os.path.join(self.testdir, 'wd') utils.mkdir(wdir) odir = os.path.join(self.testdir, 'my_levs') topof = utils.get_demo_file('srtm_oetztal.tif') run_prepro_levels(rgi_version=None, rgi_reg='11', border=20, output_folder=odir, working_dir=wdir, is_test=True, test_rgidf=rgidf, test_intersects_file=inter, test_topofile=topof) df = pd.read_csv(os.path.join(odir, 'RGI61', 'b_020', 'L0', 'summary', 'glacier_statistics_11.csv')) assert 'glacier_type' in df df = pd.read_csv(os.path.join(odir, 'RGI61', 'b_020', 'L1', 'summary', 'glacier_statistics_11.csv')) assert 'dem_source' in df df = pd.read_csv(os.path.join(odir, 'RGI61', 'b_020', 'L2', 'summary', 'glacier_statistics_11.csv')) assert 'main_flowline_length' in df df = pd.read_csv(os.path.join(odir, 'RGI61', 'b_020', 'L3', 'summary', 'glacier_statistics_11.csv')) assert 'inv_volume_km3' in df df = pd.read_csv(os.path.join(odir, 'RGI61', 'b_020', 'L3', 'summary', 'climate_statistics_11.csv')) assert '1945-1975_avg_prcp' in df assert os.path.isfile(os.path.join(odir, 'RGI61', 'b_020', 'package_versions.txt')) assert os.path.isdir(os.path.join(odir, 'RGI61', 'b_020', 'L1')) assert os.path.isdir(os.path.join(odir, 'RGI61', 'b_020', 'L2')) assert os.path.isdir(os.path.join(odir, 'RGI61', 'b_020', 'L3')) assert os.path.isdir(os.path.join(odir, 'RGI61', 'b_020', 'L4')) assert not os.path.isdir(os.path.join(odir, 'RGI61', 'b_020', 'L5')) # See if we can start from all levs from oggm import tasks from oggm.core.flowline import FlowlineModel cfg.PARAMS['continue_on_error'] = False rid = df.rgi_id.iloc[0] entity = rgidf.loc[rgidf.RGIId == rid].iloc[0] # L1 tarf = os.path.join(odir, 'RGI61', 'b_020', 'L1', rid[:8], rid[:11], rid + '.tar.gz') assert not os.path.isfile(tarf) gdir = oggm.GlacierDirectory(entity, from_tar=tarf) tasks.glacier_masks(gdir) with pytest.raises(FileNotFoundError): tasks.init_present_time_glacier(gdir) # L2 tarf = os.path.join(odir, 'RGI61', 'b_020', 'L2', rid[:8], rid[:11], rid + '.tar.gz') assert not os.path.isfile(tarf) gdir = oggm.GlacierDirectory(entity, from_tar=tarf) assert gdir.has_file('inversion_flowlines') with pytest.raises(FileNotFoundError): tasks.init_present_time_glacier(gdir) # L3 tarf = os.path.join(odir, 'RGI61', 'b_020', 'L3', rid[:8], rid[:11], rid + '.tar.gz') assert not os.path.isfile(tarf) gdir = oggm.GlacierDirectory(entity, from_tar=tarf) model = tasks.run_random_climate(gdir, nyears=10) assert isinstance(model, FlowlineModel) # L4 tarf = os.path.join(odir, 'RGI61', 'b_020', 'L4', rid[:8], rid[:11], rid + '.tar.gz') assert not os.path.isfile(tarf) gdir = oggm.GlacierDirectory(entity, from_tar=tarf) model = tasks.run_random_climate(gdir, nyears=10) assert isinstance(model, FlowlineModel) with pytest.raises(FileNotFoundError): tasks.init_present_time_glacier(gdir) def test_source_run(self): self.monkeypatch.setattr(oggm.utils, 'DEM_SOURCES', ['USER']) from oggm.cli.prepro_levels import run_prepro_levels # Read in the RGI file inter = gpd.read_file(utils.get_demo_file('rgi_intersect_oetztal.shp')) rgidf = gpd.read_file(utils.get_demo_file('rgi_oetztal.shp')) rgidf['RGIId'] = [rid.replace('RGI50', 'RGI60') for rid in rgidf.RGIId] inter['RGIId_1'] = [rid.replace('RGI50', 'RGI60') for rid in inter.RGIId_1] inter['RGIId_2'] = [rid.replace('RGI50', 'RGI60') for rid in inter.RGIId_2] rgidf = rgidf.iloc[:4] wdir = os.path.join(self.testdir, 'wd') utils.mkdir(wdir) odir = os.path.join(self.testdir, 'my_levs') topof = utils.get_demo_file('srtm_oetztal.tif') run_prepro_levels(rgi_version=None, rgi_reg='11', border=20, output_folder=odir, working_dir=wdir, is_test=True, test_rgidf=rgidf, test_intersects_file=inter, test_topofile=topof, dem_source='ALL') rid = rgidf.iloc[0].RGIId tarf = os.path.join(odir, 'RGI61', 'b_020', 'L1', rid[:8], rid[:8] + '.00.tar') assert os.path.isfile(tarf) tarf = os.path.join(odir, 'RGI61', 'b_020', 'L1', rid[:8], rid[:11], rid + '.tar.gz') assert not os.path.isfile(tarf) entity = rgidf.iloc[0] gdir = oggm.GlacierDirectory(entity, from_tar=tarf) assert os.path.isfile(os.path.join(gdir.dir, 'USER', 'dem.tif')) class TestBenchmarkCLI(unittest.TestCase): def setUp(self): self.testdir = os.path.join(get_test_dir(), 'tmp_benchmarks') self.reset_dir() def tearDown(self): if os.path.exists(self.testdir): shutil.rmtree(self.testdir) def reset_dir(self): utils.mkdir(self.testdir, reset=True) def test_parse_args(self): from oggm.cli import benchmark kwargs = benchmark.parse_args(['--rgi-reg', '1', '--map-border', '160']) assert 'working_dir' in kwargs assert 'output_folder' in kwargs assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '01' assert kwargs['border'] == 160 with pytest.raises(InvalidParamsError): benchmark.parse_args([]) with pytest.raises(InvalidParamsError): benchmark.parse_args(['--rgi-reg', '1']) with pytest.raises(InvalidParamsError): benchmark.parse_args(['--map-border', '160']) with TempEnvironmentVariable(OGGM_RGI_REG='1', OGGM_MAP_BORDER='160'): kwargs = benchmark.parse_args([]) assert 'working_dir' in kwargs assert 'output_folder' in kwargs assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '01' assert kwargs['border'] == 160 assert not kwargs['is_test'] with pytest.raises(InvalidParamsError): benchmark.parse_args([]) kwargs = benchmark.parse_args(['--rgi-reg', '1', '--map-border', '160', '--output', 'local/out', '--working-dir', 'local/work', ]) assert 'working_dir' in kwargs assert 'output_folder' in kwargs assert 'local' in kwargs['working_dir'] assert 'local' in kwargs['output_folder'] assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '01' assert kwargs['border'] == 160 assert not kwargs['is_test'] kwargs = benchmark.parse_args(['--rgi-reg', '1', '--map-border', '160', '--output', 'local/out', '--working-dir', 'local/work', '--test', ]) assert 'local' in kwargs['working_dir'] assert 'local' in kwargs['output_folder'] assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '01' assert kwargs['border'] == 160 assert kwargs['is_test'] kwargs = benchmark.parse_args(['--rgi-reg', '1', '--map-border', '160', '--output', '/local/out', '--working-dir', '/local/work', ]) assert 'tests' not in kwargs['working_dir'] assert 'tests' not in kwargs['output_folder'] assert 'local' in kwargs['working_dir'] assert 'local' in kwargs['output_folder'] assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '01' assert kwargs['border'] == 160 with TempEnvironmentVariable(OGGM_RGI_REG='12', OGGM_MAP_BORDER='120', OGGM_OUTDIR='local/out', OGGM_WORKDIR='local/work', ): kwargs = benchmark.parse_args([]) assert 'local' in kwargs['working_dir'] assert 'local' in kwargs['output_folder'] assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '12' assert kwargs['border'] == 120 with TempEnvironmentVariable(OGGM_RGI_REG='12', OGGM_MAP_BORDER='120', OGGM_OUTDIR='/local/out', OGGM_WORKDIR='/local/work', ): kwargs = benchmark.parse_args([]) assert 'local' in kwargs['working_dir'] assert 'local' in kwargs['output_folder'] assert kwargs['rgi_version'] is None assert kwargs['rgi_reg'] == '12' assert kwargs['border'] == 120 def test_full_run(self): from oggm.cli.benchmark import run_benchmark # Read in the RGI file inter = gpd.read_file(utils.get_demo_file('rgi_intersect_oetztal.shp')) rgidf = gpd.read_file(utils.get_demo_file('rgi_oetztal.shp')) rgidf['RGIId'] = [rid.replace('RGI50', 'RGI60') for rid in rgidf.RGIId] inter['RGIId_1'] = [rid.replace('RGI50', 'RGI60') for rid in inter.RGIId_1] inter['RGIId_2'] = [rid.replace('RGI50', 'RGI60') for rid in inter.RGIId_2] cru_file = utils.get_demo_file('cru_ts3.23.1901.2014.tmp.dat.nc') wdir = os.path.join(self.testdir, 'wd') utils.mkdir(wdir) odir = os.path.join(self.testdir, 'my_levs') topof = utils.get_demo_file('srtm_oetztal.tif') run_benchmark(rgi_version=None, rgi_reg='11', border=80, output_folder=odir, working_dir=wdir, is_test=True, test_rgidf=rgidf, test_intersects_file=inter, test_topofile=topof) df = pd.read_csv(os.path.join(odir, 'benchmarks_b080.csv'), index_col=0) assert len(df) > 15 def touch(path): """Equivalent to linux's touch""" with open(path, 'a'): os.utime(path, None) return path def make_fake_zipdir(dir_path, fakefile=None, base_dir=None, archiv='zip', extension='.zip'): """Creates a directory with a file in it if asked to, then compresses it""" utils.mkdir(dir_path) if fakefile: touch(os.path.join(dir_path, fakefile)) shutil.make_archive(dir_path, archiv, dir_path, base_dir) return dir_path + extension class FakeDownloadManager(): """We mess around with oggm internals, so the last we can do is to try to keep things clean after the tests.""" def __init__(self, func_name, new_func): self.func_name = func_name self.new_func = new_func self._store = getattr(_downloads, func_name) def __enter__(self): self._store = getattr(_downloads, self.func_name) setattr(_downloads, self.func_name, self.new_func) def __exit__(self, *args): setattr(_downloads, self.func_name, self._store) class TestFakeDownloads(unittest.TestCase): def setUp(self): self.dldir = os.path.join(get_test_dir(), 'tmp_download') utils.mkdir(self.dldir) # Get the path to the file before we mess around self.dem3_testfile = utils.get_demo_file('T10.zip') cfg.initialize() cfg.PATHS['dl_cache_dir'] = os.path.join(self.dldir, 'dl_cache') cfg.PATHS['working_dir'] = os.path.join(self.dldir, 'wd') cfg.PATHS['tmp_dir'] = os.path.join(self.dldir, 'extract') cfg.PATHS['rgi_dir'] = os.path.join(self.dldir, 'rgi_test') self.reset_dir() def tearDown(self): if os.path.exists(self.dldir): shutil.rmtree(self.dldir) def reset_dir(self): if os.path.exists(self.dldir): shutil.rmtree(self.dldir) utils.mkdir(self.dldir) utils.mkdir(cfg.PATHS['dl_cache_dir']) utils.mkdir(cfg.PATHS['working_dir']) utils.mkdir(cfg.PATHS['tmp_dir']) utils.mkdir(cfg.PATHS['rgi_dir']) def prepare_verify_test(self, valid_size=True, valid_crc32=True, reset_dl_dict=True): self.reset_dir() cfg.PARAMS['dl_verify'] = True if reset_dl_dict: cfg.DL_VERIFIED.clear() tgt_path = os.path.join(cfg.PATHS['dl_cache_dir'], 'test.com', 'test.txt') file_size = 1024 file_data = os.urandom(file_size) file_sha256 = hashlib.sha256() file_sha256.update(file_data) utils.mkdir(os.path.dirname(tgt_path)) with open(tgt_path, 'wb') as f: f.write(file_data) if not valid_size: file_size += 1 if not valid_crc32: file_sha256.update(b'1234ABCD') file_sha256 = file_sha256.digest() data = utils.get_dl_verify_data('cluster.klima.uni-bremen.de') s = pd.Series({'size': file_size, 'sha256': file_sha256}, name='test.txt') data = data.append(s) cfg.DATA['dl_verify_data_test.com'] = data return 'https://test.com/test.txt' def test_dl_verify(self): def fake_down(dl_func, cache_path): assert False with FakeDownloadManager('_call_dl_func', fake_down): url = self.prepare_verify_test(True, True) utils.oggm_urlretrieve(url) url = self.prepare_verify_test(False, True) with self.assertRaises(DownloadVerificationFailedException): utils.oggm_urlretrieve(url) url = self.prepare_verify_test(True, False) with self.assertRaises(DownloadVerificationFailedException): utils.oggm_urlretrieve(url) url = self.prepare_verify_test(False, False) with self.assertRaises(DownloadVerificationFailedException): utils.oggm_urlretrieve(url) def test_github_no_internet(self): self.reset_dir() cache_dir = cfg.CACHE_DIR try: cfg.CACHE_DIR = os.path.join(self.dldir, 'cache') def fake_down(dl_func, cache_path): # This should never be called, if it still is assert assert False with FakeDownloadManager('_call_dl_func', fake_down): with self.assertRaises(utils.NoInternetException): tmp = cfg.PARAMS['has_internet'] cfg.PARAMS['has_internet'] = False try: utils.download_oggm_files() finally: cfg.PARAMS['has_internet'] = tmp finally: cfg.CACHE_DIR = cache_dir def test_rgi(self): # Make a fake RGI file rgi_dir = os.path.join(self.dldir, 'rgi50') utils.mkdir(rgi_dir) make_fake_zipdir(os.path.join(rgi_dir, '01_rgi50_Region'), fakefile='test.txt') rgi_f = make_fake_zipdir(rgi_dir, fakefile='000_rgi50_manifest.txt') def down_check(url, *args, **kwargs): expected = 'http://www.glims.org/RGI/rgi50_files/rgi50.zip' self.assertEqual(url, expected) return rgi_f with FakeDownloadManager('_progress_urlretrieve', down_check): rgi = utils.get_rgi_dir(version='5') assert os.path.isdir(rgi) assert os.path.exists(os.path.join(rgi, '000_rgi50_manifest.txt')) assert os.path.exists(os.path.join(rgi, '01_rgi50_Region', 'test.txt')) # Make a fake RGI file rgi_dir = os.path.join(self.dldir, 'rgi60') utils.mkdir(rgi_dir) make_fake_zipdir(os.path.join(rgi_dir, '01_rgi60_Region'), fakefile='01_rgi60_Region.shp') rgi_f = make_fake_zipdir(rgi_dir, fakefile='000_rgi60_manifest.txt') def down_check(url, *args, **kwargs): expected = 'http://www.glims.org/RGI/rgi60_files/00_rgi60.zip' self.assertEqual(url, expected) return rgi_f with FakeDownloadManager('_progress_urlretrieve', down_check): rgi = utils.get_rgi_dir(version='6') assert os.path.isdir(rgi) assert os.path.exists(os.path.join(rgi, '000_rgi60_manifest.txt')) assert os.path.exists(os.path.join(rgi, '01_rgi60_Region', '01_rgi60_Region.shp')) with FakeDownloadManager('_progress_urlretrieve', down_check): rgi_f = utils.get_rgi_region_file('01', version='6') assert os.path.exists(rgi_f) assert '01_rgi60_Region.shp' in rgi_f def test_rgi_intersects(self): # Make a fake RGI file rgi_dir = os.path.join(self.dldir, 'RGI_V50_Intersects') utils.mkdir(rgi_dir) make_fake_zipdir(os.path.join(rgi_dir), fakefile='Intersects_OGGM_Manifest.txt') make_fake_zipdir(os.path.join(rgi_dir, '11_rgi50_CentralEurope'), fakefile='intersects_11_rgi50_CentralEurope.shp') make_fake_zipdir(os.path.join(rgi_dir, '00_rgi50_AllRegs'), fakefile='intersects_rgi50_AllRegs.shp') rgi_f = make_fake_zipdir(rgi_dir) def down_check(url, *args, **kwargs): expected = ('https://cluster.klima.uni-bremen.de/data/rgi/' + 'RGI_V50_Intersects.zip') self.assertEqual(url, expected) return rgi_f with FakeDownloadManager('_progress_urlretrieve', down_check): rgi = utils.get_rgi_intersects_dir(version='5') utils.get_rgi_intersects_region_file('11', version='5') utils.get_rgi_intersects_region_file('00', version='5') assert os.path.isdir(rgi) assert os.path.exists(os.path.join(rgi, 'Intersects_OGGM_Manifest.txt')) # Make a fake RGI file rgi_dir = os.path.join(self.dldir, 'RGI_V60_Intersects') utils.mkdir(rgi_dir) make_fake_zipdir(os.path.join(rgi_dir), fakefile='Intersects_OGGM_Manifest.txt') rgi_f = make_fake_zipdir(rgi_dir) def down_check(url, *args, **kwargs): expected = ('https://cluster.klima.uni-bremen.de/data/rgi/' + 'RGI_V60_Intersects.zip') self.assertEqual(url, expected) return rgi_f with FakeDownloadManager('_progress_urlretrieve', down_check): rgi = utils.get_rgi_intersects_dir(version='6') assert os.path.isdir(rgi) assert os.path.exists(os.path.join(rgi, 'Intersects_OGGM_Manifest.txt')) def test_srtm(self): # Make a fake topo file tf = make_fake_zipdir(os.path.join(self.dldir, 'srtm_39_03'), fakefile='srtm_39_03.tif') def down_check(url, *args, **kwargs): expected = ('http://srtm.csi.cgiar.org/wp-content/uploads/files/' 'srtm_5x5/TIFF/srtm_39_03.zip') self.assertEqual(url, expected) return tf with FakeDownloadManager('_progress_urlretrieve', down_check): of, source = utils.get_topo_file([11.3, 11.3], [47.1, 47.1]) assert os.path.exists(of[0]) assert source == 'SRTM' def test_nasadem(self): # Make a fake topo file tf = make_fake_zipdir(os.path.join(self.dldir, 'NASADEM_HGT_n47e011'), fakefile='n47e011.hgt') def down_check(url, *args, **kwargs): expected = ('https://e4ftl01.cr.usgs.gov/MEASURES/NASADEM_HGT.001/' '2000.02.11/NASADEM_HGT_n47e011.zip') self.assertEqual(url, expected) return tf with FakeDownloadManager('_progress_urlretrieve', down_check): of, source = utils.get_topo_file([11.3, 11.3], [47.1, 47.1], source='NASADEM') assert os.path.exists(of[0]) assert source == 'NASADEM' def test_dem3(self): def down_check(url, *args, **kwargs): expected = 'http://viewfinderpanoramas.org/dem3/T10.zip' self.assertEqual(url, expected) return self.dem3_testfile with FakeDownloadManager('_progress_urlretrieve', down_check): of, source = utils.get_topo_file([-120.2, -120.2], [76.8, 76.8]) assert os.path.exists(of[0]) assert source == 'DEM3' def down_check(url, *args, **kwargs): expected = ('https://cluster.klima.uni-bremen.de/~oggm/dem/' 'DEM3_MERGED/ISL.tif') self.assertEqual(url, expected) return self.dem3_testfile with FakeDownloadManager('_progress_urlretrieve', down_check): of, source = utils.get_topo_file(-22.26, 66.16, source='DEM3') assert os.path.exists(of[0]) assert source == 'DEM3' def test_mapzen(self): # Make a fake topo file tf = touch(os.path.join(self.dldir, 'file.tif')) def down_check(url, *args, **kwargs): expected = ('http://s3.amazonaws.com/elevation-tiles-prod/' 'geotiff/10/170/160.tif') self.assertEqual(url, expected) return tf with FakeDownloadManager('_progress_urlretrieve', down_check): of, source = utils.get_topo_file([-120.2, -120.2], [76.8, 76.8], source='MAPZEN', dx_meter=100) assert os.path.exists(of[0]) assert source == 'MAPZEN' def test_aw3d30(self): # Make a fake topo file deep_path = os.path.join(self.dldir, 'N049W006', 'N049W006') utils.mkdir(deep_path) upper_path = os.path.dirname(deep_path) fakefile = os.path.join(deep_path, 'N049W006_AVE_DSM.tif') tf = make_fake_zipdir(upper_path, fakefile=fakefile, archiv='gztar', extension='.tar.gz') def down_check(url, *args, **kwargs): expected = ('ftp://ftp.eorc.jaxa.jp/pub/ALOS/ext1/AW3D30/' + 'release_v1804/N045W010/N049W006.tar.gz') self.assertEqual(expected, url) return tf with FakeDownloadManager('_progress_urlretrieve', down_check): of, source = utils.get_topo_file([-5.3, -5.2], [49.5, 49.6], source='AW3D30') assert os.path.exists(of[0]) assert source == 'AW3D30' def test_copdem(self): # Make a fake topo file deep_path = os.path.join(self.dldir, 'DEM1_SAR_DGE_90_20110517T170701_20140817T170' '857_ADS_000000_4723.DEM', 'Copernicus_DSM_30_N46_00_E010_00', 'DEM') utils.mkdir(deep_path) upper_path = os.path.dirname(os.path.dirname(deep_path)) fakefile = os.path.join(deep_path, 'Copernicus_DSM_30_N46_00_E010_00_DEM.tif') tf = make_fake_zipdir(upper_path, fakefile=fakefile, base_dir='Copernicus_DSM_30_N46_00_E010_00', archiv='tar', extension='.tar') def down_check(url, *args, **kwargs): expected = ('ftps://cdsdata.copernicus.eu:990/datasets/' 'COP-DEM_GLO-90-DGED/2019_1/' 'DEM1_SAR_DGE_90_20110517T170701_20140817T170857_ADS_' '000000_4723.DEM.tar') self.assertEqual(expected, url) return tf with FakeDownloadManager('download_with_authentication', down_check): of, source = utils.get_topo_file([10.5, 10.8], [46.6, 46.8], source='COPDEM') assert os.path.exists(of[0]) assert source == 'COPDEM' def test_ramp(self): def down_check(url, *args, **kwargs): expected = 'AntarcticDEM_wgs84.tif' self.assertEqual(url, expected) return 'yo' with FakeDownloadManager('_download_topo_file_from_cluster', down_check): of, source = utils.get_topo_file([-120.2, -120.2], [-88, -88], rgi_region='19', rgi_subregion='19-11') assert of[0] == 'yo' assert source == 'RAMP' def test_rema(self): # Make a fake topo file tf = touch(os.path.join(self.dldir, 'file.tif')) def down_check(url, *args, **kwargs): expected = ('https://cluster.klima.uni-bremen.de/~oggm/dem/' 'REMA_100m_v1.1/' '40_10_100m_v1.1/40_10_100m_v1.1_reg_dem.tif') self.assertEqual(expected, url) return tf with FakeDownloadManager('_progress_urlretrieve', down_check): of, source = utils.get_topo_file(-65., -69., source='REMA') assert os.path.exists(of[0]) assert source == 'REMA' with FakeDownloadManager('_progress_urlretrieve', down_check): of, source = utils.get_topo_file([-65.1, -65.], [-69.1, -69.], source='REMA') assert os.path.exists(of[0]) assert source == 'REMA' def test_alaska(self): # Make a fake topo file tf = touch(os.path.join(self.dldir, 'file.tif')) def down_check(url, *args, **kwargs): expected = ('https://cluster.klima.uni-bremen.de/~oggm/dem/' 'Alaska_albers_V3/008_004_Alaska_albers_V3/' '008_004_Alaska_albers_V3.tif') self.assertEqual(expected, url) return tf with FakeDownloadManager('_progress_urlretrieve', down_check): of, source = utils.get_topo_file([-140.428, -140.428], [60.177, 60.177], source='ALASKA') assert os.path.exists(of[0]) assert source == 'ALASKA' def test_arcticdem(self): # Make a fake topo file tf = touch(os.path.join(self.dldir, 'file.tif')) def down_check(url, *args, **kwargs): expected = ('https://cluster.klima.uni-bremen.de/~oggm/dem/' 'ArcticDEM_100m_v3.0/' '14_52_100m_v3.0/14_52_100m_v3.0_reg_dem.tif') self.assertEqual(expected, url) return tf with FakeDownloadManager('_progress_urlretrieve', down_check): of, source = utils.get_topo_file(-21.93, 64.13, source='ARCTICDEM') assert os.path.exists(of[0]) assert source == 'ARCTICDEM' with FakeDownloadManager('_progress_urlretrieve', down_check): of, source = utils.get_topo_file([-21.93, -21.92], [64.13, 64.14], source='ARCTICDEM') assert os.path.exists(of[0]) assert source == 'ARCTICDEM' def test_gimp(self): def down_check(url, *args, **kwargs): expected = 'gimpdem_90m_v01.1.tif' self.assertEqual(url, expected) return 'yo' with FakeDownloadManager('_download_topo_file_from_cluster', down_check): of, source = utils.get_topo_file([-120.2, -120.2], [-88, -88], rgi_region=5) assert of[0] == 'yo' assert source == 'GIMP' def test_aster(self): # Make a fake topo file tf = make_fake_zipdir(os.path.join(self.dldir, 'ASTGTMV003_S88W121'), fakefile='ASTGTMV003_S89W121_dem.tif') def down_check(url, *args, **kwargs): expect = ('https://e4ftl01.cr.usgs.gov/ASTER_B/ASTT/ASTGTM.003/' + '2000.03.01/ASTGTMV003_S89W121.zip') self.assertEqual(url, expect) return tf with FakeDownloadManager('download_with_authentication', down_check): of, source = utils.get_topo_file([-120.2, -120.2], [-88.1, -88.9], source='ASTER') assert os.path.exists(of[0]) assert source == 'ASTER' def test_tandem(self): # Make a fake topo file tf = make_fake_zipdir(os.path.join(self.dldir, 'TDM1_DEM__30_N60W146'), fakefile='TDM1_DEM__30_N60W146.tif') def down_check(url, *args, **kwargs): expect = ('https://download.geoservice.dlr.de/TDM90/files/N60/' + 'W140/' + 'TDM1_DEM__30_N60W146.zip') self.assertEqual(url, expect) return tf with FakeDownloadManager('download_with_authentication', down_check): of, source = utils.get_topo_file([-145.2, -145.3], [60.1, 60.9], source='TANDEM') assert os.path.exists(of[0]) assert source == 'TANDEM' def test_tandem_invalid(self): # If the TanDEM-X tile does not exist, a invalid file is created. # See https://github.com/OGGM/oggm/issues/893 for more details # Make a fake topo file tf = {'TDM1_DEM__30_N60W144.zip': make_fake_zipdir(os.path.join(self.dldir, 'TDM1_DEM__30_N60W144'), fakefile='TDM1_DEM__30_N60W146.tif'), 'TDM1_DEM__30_N60W145.zip': touch(os.path.join(self.dldir, 'TDM1_DEM__30_N60W145.zip')), 'TDM1_DEM__30_N60W146.zip': make_fake_zipdir(os.path.join(self.dldir, 'TDM1_DEM__30_N60W146'), fakefile='TDM1_DEM__30_N60W146.tif') } def down_check(url, *args, **kwargs): file = tf.get(url.split('/')[-1]) self.assertIsNotNone(file) expect = 'https://download.geoservice.dlr.de/TDM90/files/N60/W140/' expect += file.split('/')[-1].replace('tif', '.zip') self.assertEqual(url, expect) return file with FakeDownloadManager('download_with_authentication', down_check): of, source = utils.get_topo_file([-144.1, -143.9], [60.5, 60.6], source='TANDEM') self.assertTrue(len(of) == 2) self.assertTrue(os.path.exists(of[0])) self.assertTrue(os.path.exists(of[1])) self.assertTrue(source == 'TANDEM') # check files files = [os.path.basename(f) for f in of] self.assertTrue('TDM1_DEM__30_N60W144_DEM.tif' in files) self.assertTrue('TDM1_DEM__30_N60W146_DEM.tif' in files) self.assertFalse('TDM1_DEM__30_N60W145_DEM.tif' in files) def test_cmip5(self): fn = 'pr_mon_NorESM1-M_historicalNat_r1i1p1_g025.nc' def down_check(url, *args, **kwargs): expected = ('https://cluster.klima.uni-bremen.de/~nicolas/cmip5-ng' '/pr/pr_mon_NorESM1-M_historicalNat_r1i1p1_g025.nc') self.assertEqual(url, expected) return True with FakeDownloadManager('file_downloader', down_check): assert utils.get_cmip5_file(fn) fn = 'tas_mon_CCSM4_historicalNat_r1i1p1_g025.nc' def down_check(url, *args, **kwargs): expected = ('https://cluster.klima.uni-bremen.de/~nicolas/cmip5-ng' '/tas/tas_mon_CCSM4_historicalNat_r1i1p1_g025.nc') self.assertEqual(url, expected) return True with FakeDownloadManager('file_downloader', down_check): assert utils.get_cmip5_file(fn) class TestDataFiles(unittest.TestCase): def setUp(self): self.dldir = os.path.join(get_test_dir(), 'tmp_download') utils.mkdir(self.dldir) cfg.initialize() cfg.PATHS['dl_cache_dir'] = os.path.join(self.dldir, 'dl_cache') cfg.PATHS['working_dir'] = os.path.join(self.dldir, 'wd') cfg.PATHS['tmp_dir'] = os.path.join(self.dldir, 'extract') self.reset_dir() def tearDown(self): if os.path.exists(self.dldir): shutil.rmtree(self.dldir) def reset_dir(self): if os.path.exists(self.dldir): shutil.rmtree(self.dldir) utils.mkdir(cfg.PATHS['dl_cache_dir']) utils.mkdir(cfg.PATHS['working_dir']) utils.mkdir(cfg.PATHS['tmp_dir']) def test_download_demo_files(self): f = utils.get_demo_file('Hintereisferner_RGI5.shp') self.assertTrue(os.path.exists(f)) sh = salem.read_shapefile(f) self.assertTrue(hasattr(sh, 'geometry')) # Data files cfg.initialize() lf, df = utils.get_wgms_files() self.assertTrue(os.path.exists(df)) lf = utils.get_glathida_file() self.assertTrue(os.path.exists(lf)) def test_file_extractor(self): tmp_file = os.path.join(cfg.PATHS['working_dir'], 'f.nc') touch(tmp_file) # writing file to a zipfile tmp_comp = os.path.join(cfg.PATHS['working_dir'], 'f.nc.zip') import zipfile with zipfile.ZipFile(tmp_comp, 'w') as zip: zip.write(tmp_file, arcname='f.nc') of = utils.file_extractor(tmp_comp) assert of == os.path.join(cfg.PATHS['tmp_dir'], 'f.nc') os.remove(of) # writing file to a gzip tmp_comp = os.path.join(cfg.PATHS['working_dir'], 'f.nc.gz') import gzip with gzip.GzipFile(tmp_comp, 'w') as zip: with open(tmp_file, 'rb') as f_in: shutil.copyfileobj(f_in, zip) of = utils.file_extractor(tmp_comp) assert of == os.path.join(cfg.PATHS['tmp_dir'], 'f.nc') os.remove(of) # writing file to a tar.gz tmp_comp = os.path.join(cfg.PATHS['working_dir'], 'f.nc.tar.gz') import tarfile with tarfile.open(tmp_comp, 'w:gz') as zip: zip.add(tmp_file, arcname='f.nc') of = utils.file_extractor(tmp_comp) assert of == os.path.join(cfg.PATHS['tmp_dir'], 'f.nc') os.remove(of) # Raise error if nothing to extract with pytest.raises(InvalidParamsError): utils.file_extractor(tmp_file) def test_srtmzone(self): z = utils.srtm_zone(lon_ex=[-112, -112], lat_ex=[57, 57]) self.assertTrue(len(z) == 1) self.assertEqual('14_01', z[0]) z = utils.srtm_zone(lon_ex=[-72, -73], lat_ex=[-52, -53]) self.assertTrue(len(z) == 1) self.assertEqual('22_23', z[0]) # Alps ref = sorted(['39_04', '38_03', '38_04', '39_03']) z = utils.srtm_zone(lon_ex=[6, 14], lat_ex=[41, 48]) self.assertTrue(len(z) == 4) self.assertEqual(ref, z) def test_tandemzone(self): z = utils.tandem_zone(lon_ex=[-112.1, -112.1], lat_ex=[-57.1, -57.1]) self.assertTrue(len(z) == 1) self.assertEqual('S58/W110/TDM1_DEM__30_S58W113', z[0]) z = utils.tandem_zone(lon_ex=[71, 71], lat_ex=[52, 52]) self.assertTrue(len(z) == 1) self.assertEqual('N52/E070/TDM1_DEM__30_N52E071', z[0]) z = utils.tandem_zone(lon_ex=[71, 71], lat_ex=[62, 62]) self.assertTrue(len(z) == 1) self.assertEqual('N62/E070/TDM1_DEM__30_N62E070', z[0]) z = utils.tandem_zone(lon_ex=[71, 71], lat_ex=[-82.1, -82.1]) self.assertTrue(len(z) == 1) self.assertEqual('S83/E060/TDM1_DEM__30_S83E068', z[0]) ref = sorted(['N00/E000/TDM1_DEM__30_N00E000', 'N00/E000/TDM1_DEM__30_N00E001', 'N00/W000/TDM1_DEM__30_N00W001', 'N00/W000/TDM1_DEM__30_N00W002', 'N01/E000/TDM1_DEM__30_N01E000', 'N01/E000/TDM1_DEM__30_N01E001', 'N01/W000/TDM1_DEM__30_N01W001', 'N01/W000/TDM1_DEM__30_N01W002', 'S01/E000/TDM1_DEM__30_S01E000', 'S01/E000/TDM1_DEM__30_S01E001', 'S01/W000/TDM1_DEM__30_S01W001', 'S01/W000/TDM1_DEM__30_S01W002', 'S02/E000/TDM1_DEM__30_S02E000', 'S02/E000/TDM1_DEM__30_S02E001', 'S02/W000/TDM1_DEM__30_S02W001', 'S02/W000/TDM1_DEM__30_S02W002' ]) z = utils.tandem_zone(lon_ex=[-1.3, 1.4], lat_ex=[-1.3, 1.4]) self.assertTrue(len(z) == len(ref)) self.assertEqual(ref, z) z = utils.tandem_zone(lon_ex=[-144.1, -143.9], lat_ex=[60.5, 60.6]) self.assertTrue(len(z) == 3) self.assertEqual('N60/W140/TDM1_DEM__30_N60W145', z[1]) def test_asterzone(self): z = utils.aster_zone(lon_ex=[137.5, 137.5], lat_ex=[-72.5, -72.5]) self.assertTrue(len(z) == 1) self.assertEqual('ASTGTMV003_S73E137', z[0]) z = utils.aster_zone(lon_ex=[-95.5, -95.5], lat_ex=[30.5, 30.5]) self.assertTrue(len(z) == 1) self.assertEqual('ASTGTMV003_N30W096', z[0]) z = utils.aster_zone(lon_ex=[-96.5, -94.5], lat_ex=[30.5, 30.5]) self.assertTrue(len(z) == 3) self.assertEqual('ASTGTMV003_N30W095', z[0]) self.assertEqual('ASTGTMV003_N30W096', z[1]) self.assertEqual('ASTGTMV003_N30W097', z[2]) def test_nasazone(self): z = utils.nasadem_zone(lon_ex=[137.5, 137.5], lat_ex=[-72.5, -72.5]) self.assertTrue(len(z) == 1) self.assertEqual('s73e137', z[0]) z = utils.nasadem_zone(lon_ex=[-95.5, -95.5], lat_ex=[30.5, 30.5]) self.assertTrue(len(z) == 1) self.assertEqual('n30w096', z[0]) z = utils.nasadem_zone(lon_ex=[-96.5, -94.5], lat_ex=[30.5, 30.5]) self.assertTrue(len(z) == 3) self.assertEqual('n30w095', z[0]) self.assertEqual('n30w096', z[1]) self.assertEqual('n30w097', z[2]) def test_mapzen_zone(self): z = utils.mapzen_zone(lon_ex=[137.5, 137.5], lat_ex=[45, 45], zoom=10) self.assertTrue(len(z) == 1) self.assertEqual('10/903/368.tif', z[0]) z = utils.mapzen_zone(lon_ex=[137.5, 137.5], lat_ex=[45, 45], dx_meter=110) self.assertTrue(len(z) == 1) self.assertEqual('10/903/368.tif', z[0]) z = utils.mapzen_zone(lon_ex=[137.5, 137.5], lat_ex=[-45, -45], zoom=10) self.assertTrue(len(z) == 1) self.assertEqual('10/903/655.tif', z[0]) z = utils.mapzen_zone(lon_ex=[137.5, 137.5], lat_ex=[-45, -45], dx_meter=110) self.assertTrue(len(z) == 1) self.assertEqual('10/903/655.tif', z[0]) # Check the minimum zoom level dx = 200 for lat in np.arange(10) * 10: z = utils.mapzen_zone(lon_ex=[137.5, 137.5], lat_ex=[lat, lat], dx_meter=dx) assert int(z[0].split('/')[0]) > 9 z = utils.mapzen_zone(lon_ex=[181, 181], lat_ex=[-lat, -lat], dx_meter=dx) assert int(z[0].split('/')[0]) > 9 def test_dem3_viewpano_zone(self): lon_ex = -22.26 lat_ex = 66.16 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['ISL'] lon_ex = 17 lat_ex = 80 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['SVALBARD'] lon_ex = -8 lat_ex = 71 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['JANMAYEN'] lon_ex = 60 lat_ex = 80 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['FJ'] lon_ex = -7 lat_ex = 62 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['FAR'] lon_ex = 19 lat_ex = 74.5 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['BEAR'] lon_ex = -1 lat_ex = 60.5 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['SHL'] lon_ex = -30.733021 lat_ex = 82.930238 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['GL-North'] lon_ex = -52 lat_ex = 70 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['GL-West'] lon_ex = -43 lat_ex = 60 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['GL-South'] lon_ex = -24.7 lat_ex = 69.8 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['GL-East'] lon_ex = -22.26 lat_ex = 66.16 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['ISL'] lon_ex = -127.592802 lat_ex = -74.479523 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['01-15'] lat_ex = -72.383226 lon_ex = -60.648126 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['16-30'] lat_ex = -67.993527 lon_ex = 65.482151 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['31-45'] lat_ex = -71.670896 lon_ex = 166.878916 assert utils.dem3_viewpano_zone(lon_ex, lat_ex) == ['46-60'] # normal tile z = utils.dem3_viewpano_zone([-179., -178.], [65., 65.]) self.assertTrue(len(z) == 1) self.assertEqual('Q01', z[0]) # normal tile z = utils.dem3_viewpano_zone([107, 107], [69, 69]) self.assertTrue(len(z) == 1) self.assertEqual('R48', z[0]) # Alps ref = sorted(['K31', 'K32', 'K33', 'L31', 'L32', 'L33', 'M31', 'M32', 'M33']) z = utils.dem3_viewpano_zone([6, 14], [41, 48]) self.assertTrue(len(z) == 9) self.assertEqual(ref, z) def test_copdemzone(self): z = utils.copdem_zone(lat_ex=[-9.8, -9.5], lon_ex=[-77.6, -77.3]) self.assertTrue(len(z) == 1) self.assertEqual(('DEM1_SAR_DGE_90_20110427T104941_20140819T105300_' 'ADS_000000_2733.DEM.tar'), z[0][0]) self.assertEqual('Copernicus_DSM_30_S10_00_W078_00', z[0][1]) z = utils.copdem_zone(lat_ex=[46.37, 46.59], lon_ex=[7.89, 8.12]) self.assertTrue(len(z) == 2) self.assertTrue('Copernicus_DSM_30_N46_00_E008_00' in [z[0][1], z[1][1]]) self.assertTrue('Copernicus_DSM_30_N46_00_E007_00' in [z[0][1], z[1][1]]) # we want an error if copdem does not find all or any self.assertRaises(InvalidDEMError, utils.copdem_zone, lat_ex=[0, 1], lon_ex=[0, 1]) def test_is_dem_source_available(self): assert utils.is_dem_source_available('SRTM', [11, 11], [47, 47]) assert utils.is_dem_source_available('GIMP', [-25, -25], [71, 71]) assert not utils.is_dem_source_available('GIMP', [11, 11], [47, 47]) assert utils.is_dem_source_available('ARCTICDEM', [-25, -25], [71, 71]) assert utils.is_dem_source_available('RAMP', [-25, -25], [-71, -71]) assert utils.is_dem_source_available('REMA', [-25, -25], [-71, -71]) assert not utils.is_dem_source_available('AW3D30', [5, 5], [-60, -60]) for s in ['TANDEM', 'AW3D30', 'MAPZEN', 'DEM3', 'ASTER', 'AW3D30']: assert utils.is_dem_source_available(s, [11, 11], [47, 47]) def test_find_dem_zone(self): # Somewhere in the Alps: SRTM assert utils.default_dem_source(11, 47) == 'SRTM' # High Lats: DEM3 assert utils.default_dem_source(11, 65) == 'DEM3' # Eastern russia: DEM3 assert utils.default_dem_source(170.1, 59.1) == 'DEM3' # Greenland assert utils.default_dem_source(0, 0, rgi_region='5') == 'GIMP' # Antarctica with pytest.raises(InvalidParamsError): utils.default_dem_source(0, 0, rgi_region='19') assert utils.default_dem_source(0, 0, rgi_region='19', rgi_subregion='19-06') == 'RAMP' assert utils.default_dem_source(0, 0, rgi_region='19', rgi_subregion='19-01') == 'DEM3' def test_lrufilecache(self): f1 = os.path.join(self.dldir, 'f1.txt') f2 = os.path.join(self.dldir, 'f2.txt') f3 = os.path.join(self.dldir, 'f3.txt') open(f1, 'a').close() open(f2, 'a').close() open(f3, 'a').close() assert os.path.exists(f1) lru = utils.LRUFileCache(maxsize=2) lru.append(f1) assert os.path.exists(f1) lru.append(f2) assert os.path.exists(f1) lru.append(f3) assert not os.path.exists(f1) assert os.path.exists(f2) lru.append(f2) assert os.path.exists(f2) open(f1, 'a').close() lru = utils.LRUFileCache(l0=[f2, f3], maxsize=2) assert os.path.exists(f1) assert os.path.exists(f2) assert os.path.exists(f3) lru.append(f1) assert os.path.exists(f1) assert not os.path.exists(f2) assert os.path.exists(f3) def test_lruhandler(self): # initiate some files in empty directory self.reset_dir() foo = [] bar = [] for i, e in itertools.product(range(1, 4), ['foo', 'bar']): f = os.path.join(self.dldir, 'f{}.{}'.format(i, e)) open(f, 'a').close() (foo.append(f) if e == 'foo' else bar.append(f)) time.sleep(0.1) # init handler for dldir and ending 'foo' lru_foo = cfg.get_lru_handler(self.dldir, ending='.foo') # no maxsize specified: use default self.assertTrue(lru_foo.maxsize == cfg.PARAMS['lru_maxsize']) # all files should exist self.assertTrue(all([os.path.exists(f) for f in foo])) # init handler for dldir and ending bar cfg.get_lru_handler(self.dldir, maxsize=3, ending='.bar') # all files should exist self.assertTrue(all([os.path.exists(b) for b in bar])) # update foo handler, decresing maxsize should delete first file lru_foo2 = cfg.get_lru_handler(self.dldir, maxsize=2, ending='.foo') self.assertFalse(os.path.exists(foo[0])) self.assertTrue(os.path.exists(foo[1])) self.assertTrue(os.path.exists(foo[2])) # but this should be the same handler instance as above: self.assertTrue(lru_foo is lru_foo2) self.assertTrue(lru_foo.maxsize == 2) # And bar files should not be affected by decreased cache size of foo self.assertTrue(all([os.path.exists(b) for b in bar])) @pytest.mark.download def test_srtmdownload(self): from oggm.utils._downloads import _download_srtm_file # this zone does exist and file should be small enough for download zone = '68_11' fp = _download_srtm_file(zone) self.assertTrue(os.path.exists(fp)) fp = _download_srtm_file(zone) self.assertTrue(os.path.exists(fp)) @pytest.mark.download def test_srtmdownloadfails(self): from oggm.utils._downloads import _download_srtm_file # this zone does not exist zone = '41_20' self.assertTrue(_download_srtm_file(zone) is None) @pytest.mark.download def test_nasadownload(self): from oggm.utils._downloads import _download_nasadem_file # this zone does exist and file should be small enough for download zone = 'n01e119' fp = _download_nasadem_file(zone) self.assertTrue(os.path.exists(fp)) fp = _download_nasadem_file(zone) self.assertTrue(os.path.exists(fp)) @pytest.mark.download def test_nasademdownloadfails(self): from oggm.utils._downloads import _download_nasadem_file # this zone does not exist zone = 'n80w100' self.assertTrue(_download_nasadem_file(zone) is None) @pytest.mark.download @pytest.mark.creds def test_tandemdownload(self): from oggm.utils._downloads import _download_tandem_file # this zone does exist and file should be small enough for download zone = 'N47/E010/TDM1_DEM__30_N47E011' fp = _download_tandem_file(zone) self.assertTrue(os.path.exists(fp)) fp = _download_tandem_file(zone) self.assertTrue(os.path.exists(fp)) @pytest.mark.download @pytest.mark.creds def test_tandemdownloadfails(self): from oggm.utils._downloads import _download_tandem_file # this zone does not exist, so it should return None zone = 'N47/E010/TDM1_DEM__30_NxxExxx' fp = _download_tandem_file(zone) self.assertIsNone(fp) @pytest.mark.download @pytest.mark.creds def test_copdemdownload(self): from oggm.utils._downloads import _download_copdem_file # this zone does exist and file should be small enough for download cppfile = ('DEM1_SAR_DGE_90_20110109T094723_20140326T001716_ADS_' '000000_5845.DEM.tar') tilename = 'Copernicus_DSM_30_N78_00_E102_00' fp = _download_copdem_file(cppfile, tilename) self.assertTrue(os.path.exists(fp)) @pytest.mark.creds def test_asterdownload(self): from oggm.utils._downloads import _download_aster_file # this zone does exist and file should be small enough for download zone = 'ASTGTMV003_S73E137' fp = _download_aster_file(zone) self.assertTrue(os.path.exists(fp)) @pytest.mark.download def test_mapzen_download(self): from oggm.utils._downloads import _download_mapzen_file # this zone does exist and file should be small enough for download zone = '10/903/368.tif' fp = _download_mapzen_file(zone) self.assertTrue(os.path.exists(fp)) @pytest.mark.download def test_gimp(self): fp, z = utils.get_topo_file([], [], rgi_region=5) self.assertTrue(os.path.exists(fp[0])) self.assertEqual(z, 'GIMP') @pytest.mark.download def test_iceland(self): fp, z = utils.get_topo_file([-20, -20], [65, 65]) self.assertTrue(os.path.exists(fp[0])) @pytest.mark.creds def test_asterdownloadfails(self): from oggm.utils._downloads import _download_aster_file # this zone does not exist zone = 'ASTGTMV003_S99E010' self.assertTrue(_download_aster_file(zone) is None) @pytest.mark.download def test_download_histalp(self): from oggm.shop.histalp import get_histalp_file of = get_histalp_file('tmp') self.assertTrue(os.path.exists(of)) of = get_histalp_file('pre') self.assertTrue(os.path.exists(of)) @pytest.mark.download def test_download_rgi5(self): tmp = cfg.PATHS['rgi_dir'] cfg.PATHS['rgi_dir'] = os.path.join(self.dldir, 'rgi_extract') of = utils.get_rgi_dir(version='5') of = os.path.join(of, '01_rgi50_Alaska', '01_rgi50_Alaska.shp') self.assertTrue(os.path.exists(of)) cfg.PATHS['rgi_dir'] = tmp @pytest.mark.download def test_download_rgi6(self): tmp = cfg.PATHS['rgi_dir'] cfg.PATHS['rgi_dir'] = os.path.join(self.dldir, 'rgi_extract') of = utils.get_rgi_dir(version='6') of = os.path.join(of, '01_rgi60_Alaska', '01_rgi60_Alaska.shp') self.assertTrue(os.path.exists(of)) cfg.PATHS['rgi_dir'] = tmp @pytest.mark.download def test_download_rgi61(self): tmp = cfg.PATHS['rgi_dir'] cfg.PATHS['rgi_dir'] = os.path.join(self.dldir, 'rgi_extract') of = utils.get_rgi_dir(version='61') of = os.path.join(of, '01_rgi61_Alaska', '01_rgi61_Alaska.shp') self.assertTrue(os.path.exists(of)) cfg.PATHS['rgi_dir'] = tmp @pytest.mark.download def test_download_rgi60_intersects(self): tmp = cfg.PATHS['rgi_dir'] cfg.PATHS['rgi_dir'] = os.path.join(self.dldir, 'rgi_extract') of = utils.get_rgi_intersects_dir(version='6') of = os.path.join(of, '01_rgi60_Alaska', 'intersects_01_rgi60_Alaska.shp') self.assertTrue(os.path.exists(of)) cfg.PATHS['rgi_dir'] = tmp @pytest.mark.download def test_download_rgi61_intersects(self): tmp = cfg.PATHS['rgi_dir'] cfg.PATHS['rgi_dir'] = os.path.join(self.dldir, 'rgi_extract') of = utils.get_rgi_intersects_dir(version='61') of = os.path.join(of, '01_rgi61_Alaska', 'intersects_01_rgi61_Alaska.shp') self.assertTrue(os.path.exists(of)) cfg.PATHS['rgi_dir'] = tmp @pytest.mark.download def test_download_dem3_viewpano(self): from oggm.utils._downloads import _download_dem3_viewpano # this zone does exist and file should be small enough for download zone = 'L32' fp = _download_dem3_viewpano(zone) self.assertTrue(os.path.exists(fp)) zone = 'U44' fp = _download_dem3_viewpano(zone) self.assertTrue(os.path.exists(fp)) @pytest.mark.download def test_download_dem3_viewpano_fails(self): from oggm.utils._downloads import _download_dem3_viewpano # this zone does not exist zone = 'dummy' fp = _download_dem3_viewpano(zone) self.assertTrue(fp is None) @pytest.mark.download def test_download_aw3d30(self): from oggm.utils._downloads import _download_aw3d30_file # this zone does exist and file should be small enough for download zone = 'N000E105/N002E107' fp = _download_aw3d30_file(zone) self.assertTrue(os.path.exists(fp)) zone = 'S085W050/S081W048' fp = _download_aw3d30_file(zone) self.assertTrue(os.path.exists(fp)) @pytest.mark.download def test_download_aw3d30_fails(self): from oggm.utils._downloads import _download_aw3d30_file from urllib.error import URLError # this zone does not exist zone = 'N000E005/N000E005' self.assertRaises(URLError, _download_aw3d30_file, zone) @pytest.mark.download def test_download_cmip5(self): fn = 'pr_mon_NorESM1-M_historicalNat_r1i1p1_g025.nc' fp = utils.get_cmip5_file(fn) self.assertTrue(os.path.isfile(fp)) @pytest.mark.download def test_auto_topo(self): # Test for combine fdem, src = utils.get_topo_file([6, 14], [41, 41]) self.assertEqual(src, 'SRTM') self.assertEqual(len(fdem), 2) for fp in fdem: self.assertTrue(os.path.exists(fp)) fdem, src = utils.get_topo_file([-143, -131], [61, 61]) self.assertEqual(src, 'DEM3') self.assertEqual(len(fdem), 3) for fp in fdem: self.assertTrue(os.path.exists(fp)) @pytest.mark.download def test_from_prepro(self): # Read in the RGI file rgi_file = utils.get_demo_file('rgi_oetztal.shp') rgidf = gpd.read_file(rgi_file) rgidf['RGIId'] = [rid.replace('RGI50', 'RGI60') for rid in rgidf.RGIId] # Go - initialize working directories gdirs = workflow.init_glacier_directories(rgidf.iloc[:2], from_prepro_level=1, prepro_rgi_version='61', prepro_border=10, use_demo_glaciers=False) n_intersects = 0 for gdir in gdirs: assert gdir.has_file('dem') n_intersects += gdir.has_file('intersects') assert n_intersects > 0 workflow.execute_entity_task(tasks.glacier_masks, gdirs) class TestSkyIsFalling(unittest.TestCase): def test_projplot(self): # this caused many problems on Linux Mint distributions # this is just to be sure that on your system, everything is fine import pyproj import matplotlib.pyplot as plt pyproj.Proj(proj='latlong', datum='WGS84') plt.figure() plt.close() srs = ('+units=m +proj=lcc +lat_1=29.0 +lat_2=29.0 ' '+lat_0=29.0 +lon_0=89.8') proj_out = pyproj.Proj("EPSG:4326", preserve_units=True) proj_in = pyproj.Proj(srs, preserve_units=True) from salem.gis import transform_proj lon, lat = transform_proj(proj_in, proj_out, -2235000, -2235000) np.testing.assert_allclose(lon, 70.75731, atol=1e-5)