diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..a3dca81 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,40 @@ +name: Tests + +on: + pull_request: + branches: + - master + workflow_dispatch: + +jobs: + test: + name: Run Tests + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + + steps: + - uses: actions/checkout@v5 + + - name: Set Up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry + uses: abatilo/actions-poetry@v4 + + - name: Cache Virtual Environment + uses: actions/cache@v4 + with: + path: ./.venv + key: venv-${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }} + + - name: Install Dependencies + run: poetry install --all-extras + + - name: Run Tests + run: poetry run pytest diff --git a/poetry.lock b/poetry.lock index ed65293..1605791 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "alabaster" @@ -359,6 +359,22 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 [package.extras] toml = ["tomli"] +[[package]] +name = "cwms-python" +version = "1.0.8" +description = "Corps water management systems (CWMS) REST API for Data Retrieval of USACE water data" +optional = true +python-versions = "<4.0,>=3.9" +files = [ + {file = "cwms_python-1.0.8-py3-none-any.whl", hash = "sha256:9cdc2e6fb60be9561f680900ad2eb7d2cfda2c5c4157fc7fbb94ac3f4dd3f278"}, + {file = "cwms_python-1.0.8.tar.gz", hash = "sha256:1ff75098eec279c51a27a03431c880bf5baaf7ddc0ea3e10b6af0b4b4d35c527"}, +] + +[package.dependencies] +pandas = ">=2.1.3,<3.0.0" +requests = ">=2.31.0,<3.0.0" +requests-toolbelt = ">=1.0.0,<2.0.0" + [[package]] name = "distlib" version = "0.4.0" @@ -431,6 +447,21 @@ files = [ {file = "filelock-3.24.3.tar.gz", hash = "sha256:011a5644dc937c22699943ebbfc46e969cdde3e171470a6e40b9533e5a72affa"}, ] +[[package]] +name = "hecdss" +version = "0.1.29" +description = "Python wrapper for the HEC-DSS file database C library." +optional = true +python-versions = ">=3.8" +files = [ + {file = "hecdss-0.1.29-py3-none-any.whl", hash = "sha256:6884e5a47c98a761c3429ec4aa3ba382e8a6eb55f1d5ac40416ecb0c10757848"}, + {file = "hecdss-0.1.29.tar.gz", hash = "sha256:39289029d0ec6791e6423f0b91617b473c452fe340448452d8134011ad8f8624"}, +] + +[package.dependencies] +numpy = "*" +tzdata = "*" + [[package]] name = "identify" version = "2.6.15" @@ -831,6 +862,60 @@ files = [ {file = "nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb"}, ] +[[package]] +name = "numpy" +version = "2.0.2" +description = "Fundamental package for array computing in Python" +optional = true +python-versions = ">=3.9" +files = [ + {file = "numpy-2.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8c5713284ce4e282544c68d1c3b2c7161d38c256d2eefc93c1d683cf47683e66"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:becfae3ddd30736fe1889a37f1f580e245ba79a5855bff5f2a29cb3ccc22dd7b"}, + {file = "numpy-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da5960c3cf0df7eafefd806d4e612c5e19358de82cb3c343631188991566ccd"}, + {file = "numpy-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:496f71341824ed9f3d2fd36cf3ac57ae2e0165c143b55c3a035ee219413f3318"}, + {file = "numpy-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a61ec659f68ae254e4d237816e33171497e978140353c0c2038d46e63282d0c8"}, + {file = "numpy-2.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d731a1c6116ba289c1e9ee714b08a8ff882944d4ad631fd411106a30f083c326"}, + {file = "numpy-2.0.2-cp310-cp310-win32.whl", hash = "sha256:984d96121c9f9616cd33fbd0618b7f08e0cfc9600a7ee1d6fd9b239186d19d97"}, + {file = "numpy-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7b0be4ef08607dd04da4092faee0b86607f111d5ae68036f16cc787e250a131"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a"}, + {file = "numpy-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669"}, + {file = "numpy-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951"}, + {file = "numpy-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9"}, + {file = "numpy-2.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15"}, + {file = "numpy-2.0.2-cp311-cp311-win32.whl", hash = "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4"}, + {file = "numpy-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c"}, + {file = "numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692"}, + {file = "numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a"}, + {file = "numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c"}, + {file = "numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded"}, + {file = "numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5"}, + {file = "numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9059e10581ce4093f735ed23f3b9d283b9d517ff46009ddd485f1747eb22653c"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:423e89b23490805d2a5a96fe40ec507407b8ee786d66f7328be214f9679df6dd"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:2b2955fa6f11907cf7a70dab0d0755159bca87755e831e47932367fc8f2f2d0b"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:97032a27bd9d8988b9a97a8c4d2c9f2c15a81f61e2f21404d7e8ef00cb5be729"}, + {file = "numpy-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e795a8be3ddbac43274f18588329c72939870a16cae810c2b73461c40718ab1"}, + {file = "numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b258c385842546006213344c50655ff1555a9338e2e5e02a0756dc3e803dd"}, + {file = "numpy-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fec9451a7789926bcf7c2b8d187292c9f93ea30284802a0ab3f5be8ab36865d"}, + {file = "numpy-2.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9189427407d88ff25ecf8f12469d4d39d35bee1db5d39fc5c168c6f088a6956d"}, + {file = "numpy-2.0.2-cp39-cp39-win32.whl", hash = "sha256:905d16e0c60200656500c95b6b8dca5d109e23cb24abc701d41c02d74c6b3afa"}, + {file = "numpy-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:a3f4ab0caa7f053f6797fcd4e1e25caee367db3112ef2b6ef82d749530768c73"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7f0a0c6f12e07fa94133c8a67404322845220c06a9e80e85999afe727f7438b8"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:312950fdd060354350ed123c0e25a71327d3711584beaef30cdaa93320c392d4"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26df23238872200f63518dd2aa984cfca675d82469535dc7162dc2ee52d9dd5c"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a46288ec55ebbd58947d31d72be2c63cbf839f0a63b49cb755022310792a3385"}, + {file = "numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78"}, +] + [[package]] name = "packaging" version = "26.0" @@ -842,6 +927,105 @@ files = [ {file = "packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4"}, ] +[[package]] +name = "pandas" +version = "2.3.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = true +python-versions = ">=3.9" +files = [ + {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, + {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, + {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, + {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, + {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, + {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, + {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, + {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, + {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, + {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, + {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, + {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, + {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, + {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, + {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, + {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, + {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, + {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, + {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, + {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, + {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, + {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, + {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, + {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, + {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, + {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, + {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, + {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, + {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, + {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, + {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, + {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, + {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, + {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, + {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, + {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, + {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, + {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, + {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, + {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, + {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, + {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, + {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, + {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, + {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, + {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, + {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, + {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, + {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, + {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, + {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, + {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, + {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, + {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, + {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + [[package]] name = "pathspec" version = "1.0.4" @@ -1016,6 +1200,31 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2026.2" +description = "World timezone definitions, modern and historical" +optional = true +python-versions = "*" +files = [ + {file = "pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126"}, + {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1119,6 +1328,20 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +description = "A utility belt for advanced users of python-requests" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, + {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, +] + +[package.dependencies] +requests = ">=2.0.1,<3.0.0" + [[package]] name = "ruyaml" version = "0.91.0" @@ -1157,6 +1380,17 @@ enabler = ["pytest-enabler (>=2.2)"] test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.18.*)", "pytest-mypy"] +[[package]] +name = "six" +version = "1.17.0" +description = "Python 2 and 3 compatibility utilities" +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, +] + [[package]] name = "snowballstemmer" version = "3.0.1" @@ -1432,6 +1666,17 @@ files = [ {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, ] +[[package]] +name = "tzdata" +version = "2026.2" +description = "Provider of IANA time zone data" +optional = true +python-versions = ">=2" +files = [ + {file = "tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7"}, + {file = "tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10"}, +] + [[package]] name = "urllib3" version = "2.6.3" @@ -1505,9 +1750,11 @@ test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_it type = ["pytest-mypy"] [extras] +cda = ["cwms-python"] docs = ["sphinx", "sphinx-design", "sphinx_rtd_theme"] +dss = ["hecdss"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "f65a310a25dff39fac54e1029638b00b3c42c0b6ae74772d047d3ad932b7288c" +content-hash = "ac31607b258daeddf40546d97b5ffb17452f729487a15617404e4e249763ebaa" diff --git a/pyproject.toml b/pyproject.toml index 5e6b213..08e7333 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,12 +17,16 @@ authors = ["Hydrologic Engineering Center"] [tool.poetry.dependencies] python = "^3.9" click = "^8.1" +cwms-python = { version = "^1.0.8", optional = true } +hecdss = { version = "^0.1.29", optional = true } sphinx = { version = "^7.0", optional = true } sphinx_rtd_theme = { version = "^2.0", optional = true } sphinx-design = { version = "^0.5", optional = true } [tool.poetry.extras] docs = ["sphinx", "sphinx_rtd_theme", "sphinx-design"] +cda = ["cwms-python"] +dss = ["hecdss"] [tool.poetry.group.dev.dependencies] black = "^24.2.0" diff --git a/shef/exporters/abstract_exporter.py b/shef/exporters/abstract_exporter.py index 7cb0a8a..d2747f1 100644 --- a/shef/exporters/abstract_exporter.py +++ b/shef/exporters/abstract_exporter.py @@ -5,8 +5,6 @@ from io import BufferedRandom, StringIO from typing import Optional, TextIO, Union -import cwms # type: ignore - from shef.loaders import abstract_loader, shared diff --git a/shef/exporters/cda_exporter.py b/shef/exporters/cda_exporter.py index 72e5c8b..6438ecc 100644 --- a/shef/exporters/cda_exporter.py +++ b/shef/exporters/cda_exporter.py @@ -35,7 +35,6 @@ def __init__(self, cda_url: str, office: str): self._office = office self._cda_loader = loaders.cda_loader.CdaLoader(self.logger, sys.stdout) self._cda_loader.set_options(f"[{cda_url}][][{office}]") - self._cda_loader.make_export_transforms() def export(self, timeseries_or_group: str) -> None: """ @@ -45,35 +44,50 @@ def export(self, timeseries_or_group: str) -> None: timeseries_or_group (str): If a time series ID, export that time series; If a time series group ID, export each time series in that group """ if len(timeseries_or_group.split(".")) == 6: + self._cda_loader.make_export_transforms() timeseries_ids = [timeseries_or_group] - elif timeseries_or_group in self._cda_loader._export_groups: + else: + self._cda_loader.make_export_transforms(group_id=timeseries_or_group) + if timeseries_or_group not in self._cda_loader._export_groups: + raise shared.LoaderException( + f"Time series group [{timeseries_or_group}] not found under the SHEF Export category for office [{self._office}]" + ) timeseries_ids = self._cda_loader._export_groups[timeseries_or_group][ "timeseries" ] total_value_count: int = 0 - first = True - data = StringIO() - data.write("[") + ts_payloads: list[Any] = [] for tsid in timeseries_ids: - unit = self._cda_loader._transforms[tsid].units - ts = cwms.get_timeseries( - ts_id=tsid, - office_id=self._cda_loader._office_id, - unit=unit, - begin=self._start_time, - end=self._end_time, - ) - value_count = len(ts.json["values"]) - if value_count > 0: - if not first: - data.write(",") - data.write(json.dumps(ts.json)) - total_value_count += value_count - first = False - data.write("]") - to_unload = data.getvalue() - data.close() - if (total_value_count) > 0: + try: + unit = self._cda_loader._transforms[tsid].units + ts = cwms.get_timeseries( + ts_id=tsid, + office_id=self._cda_loader._office_id, + unit=unit, + begin=self._start_time, + end=self._end_time, + ) + except Exception as e: + self.logger.warning( + f"Skipping time series [{tsid}]: error fetching from CDA: {e}" + ) + continue + ts_json = ts.json if ts is not None else None + if not isinstance(ts_json, dict): + self.logger.warning( + f"Skipping time series [{tsid}]: CDA response is not a JSON object" + ) + continue + values = ts_json.get("values") or [] + if not values: + self.logger.info( + f"Skipping time series [{tsid}]: no values in window {self._start_time} to {self._end_time}" + ) + continue + ts_payloads.append(ts_json) + total_value_count += len(values) + to_unload = json.dumps(ts_payloads) + if total_value_count > 0: try: old_output = self._cda_loader._output self._cda_loader._output = self._output @@ -90,6 +104,7 @@ def get_groups(self) -> dict[str, str]: Returns: dict[str, str]: A dictionary of time series group descriptions keyed by time series group IDs """ + self._cda_loader.make_export_transforms() return { group: self._cda_loader._export_groups[group]["description"] for group in self._cda_loader._export_groups @@ -105,6 +120,7 @@ def get_time_series(self, group: str) -> list[str]: Returns: list[str]: The assigned time series IDs """ + self._cda_loader.make_export_transforms(group_id=group) return [ts for ts in self._cda_loader._export_groups[group]["timeseries"]] def get_unit(self, tsid: str) -> Optional[str]: @@ -117,6 +133,8 @@ def get_unit(self, tsid: str) -> Optional[str]: Returns: Optional[str]: The unit as specified in the time series alias """ + if tsid not in self._cda_loader._transforms: + self._cda_loader.make_export_transforms() return self._cda_loader._transforms[tsid].units diff --git a/shef/loaders/cda_loader.py b/shef/loaders/cda_loader.py index 501f44e..6e1df67 100644 --- a/shef/loaders/cda_loader.py +++ b/shef/loaders/cda_loader.py @@ -106,6 +106,8 @@ def __init__( self._value_error_count: int = 0 self._write_tasks: list[Coroutine[Any, Any, Any]] = [] self._configured_pe_codes = set() + self._loaded_export_group_ids: set[str] = set() + self._loaded_all_export_groups: bool = False def make_shef_transform(self, crit: dict[str, Any]) -> ShefTransform: """ @@ -462,55 +464,72 @@ def set_input(self, input_object: Union[StringIO, TextIO, str]) -> None: f"Expected TextIOWrapper or str object, got [{input_object.__class__.__name__}]" ) - def make_export_transforms(self) -> None: + def make_export_transforms(self, group_id: Optional[str] = None) -> None: if not self._office_id: raise shared.LoaderException( f"Cannot unload without office specified, use options [api_root][api_key][office]" ) - if not self._transforms: - tsids_used: dict[str, list[str]] = {} - group_list = cwms.get_timeseries_groups( - office_id=self._office_id, - include_assigned=True, - timeseries_category_like="SHEF Export", - timeseries_group_like="^.+$", - category_office_id="CWMS", - ).json - for shef_group in group_list: - group_id = shef_group["id"] - if "description" not in shef_group: - shef_group["description"] = "" - self._export_groups[group_id] = { - "description": shef_group["description"], - "timeseries": [], - } - try: - for time_series in shef_group["assigned-time-series"]: - transform = self.make_shef_transform(time_series) - transform_key = ( - f"{transform.location}.{transform.parameter_code}" - ) - self._transforms[transform_key] = transform - if transform.timeseries_id in tsids_used: - if self._logger: - self._logger.warning( - f"Tranform for time seires {transform.timeseries_id} specified in group(s) " - f"{','.join(tsids_used[transform.timeseries_id])} is/are overriden by transform specified in group {group_id}" - ) - self._export_groups[group_id]["timeseries"].append( - transform.timeseries_id - ) - tsids_used.setdefault(transform.timeseries_id, []).append( - group_id - ) - self._transforms[transform.timeseries_id] = ( - transform # to be able to retrieve by time series ID + if self._loaded_all_export_groups: + return + if group_id is None: + group_filter = "^.+$" + elif group_id in self._loaded_export_group_ids: + return + else: + group_filter = f"^{re.escape(group_id)}$" + tsids_used: dict[str, list[str]] = {} + group_list = cwms.get_timeseries_groups( + office_id=self._office_id, + include_assigned=True, + timeseries_category_like="SHEF Export", + timeseries_group_like=group_filter, + category_office_id="CWMS", + group_office_id=self._office_id, + ).json + for shef_group in group_list: + shef_group_id = shef_group["id"] + if "description" not in shef_group: + shef_group["description"] = "" + self._export_groups[shef_group_id] = { + "description": shef_group["description"], + "timeseries": [], + } + for time_series in shef_group["assigned-time-series"]: + if not time_series.get("alias-id"): + if self._logger: + self._logger.warning( + f"Skipping time series {time_series.get('timeseries-id')} in group {shef_group_id}: missing or empty alias-id" ) + continue + try: + transform = self.make_shef_transform(time_series) + transform_key = ( + f"{transform.location}.{transform.parameter_code}" + ) + self._transforms[transform_key] = transform + if transform.timeseries_id in tsids_used: + if self._logger: + self._logger.warning( + f"Tranform for time seires {transform.timeseries_id} specified in group(s) " + f"{','.join(tsids_used[transform.timeseries_id])} is/are overriden by transform specified in group {shef_group_id}" + ) + self._export_groups[shef_group_id]["timeseries"].append( + transform.timeseries_id + ) + tsids_used.setdefault(transform.timeseries_id, []).append( + shef_group_id + ) + self._transforms[transform.timeseries_id] = ( + transform # to be able to retrieve by time series ID + ) except Exception as e: if self._logger: self._logger.warning( - f"{str(e)} occurred while processing SHEF criteria for {time_series['timeseries-id']}" + f"{str(e)} occurred while processing SHEF criteria for {time_series.get('timeseries-id')}" ) + self._loaded_export_group_ids.add(shef_group_id) + if group_id is None: + self._loaded_all_export_groups = True def unload(self) -> None: """ diff --git a/shef/loaders/dss_loader.py b/shef/loaders/dss_loader.py index bd4ee7a..9c95816 100644 --- a/shef/loaders/dss_loader.py +++ b/shef/loaders/dss_loader.py @@ -6,7 +6,14 @@ from logging import Logger from typing import Any, Optional, TextIO, Union, cast -from hecdss import HecDss, IrregularTimeSeries, RegularTimeSeries # type: ignore +try: + from hecdss import HecDss, IrregularTimeSeries, RegularTimeSeries # type: ignore + HECDSS_AVAILABLE = True +except ImportError: + HECDSS_AVAILABLE = False + HecDss = None + IrregularTimeSeries = None + RegularTimeSeries = None from shef.constants import PE_CONVERSIONS from shef.loaders import abstract_loader, shared @@ -68,6 +75,8 @@ def set_options(self, options_str: Optional[str]) -> None: """ Set the sensor and parameter file names """ + if not HECDSS_AVAILABLE: + raise ImportError("The 'hecdss' library is required but not installed. Please install it to use DSS functionality.") if not options_str: raise shared.LoaderException( f"Empty options on {self.loader_name}.set_options()" @@ -617,6 +626,8 @@ def load_time_series(self) -> None: """ Store the time series to HEC-DSS file """ + if not HECDSS_AVAILABLE: + raise ImportError("The 'hecdss' library is required but not installed. Please install it to use DSS functionality.") if self._shef_value and self._time_series: if self._dss_file is None: self._dss_file = HecDss(self._dss_file_name) diff --git a/shef/shef_parser.py b/shef/shef_parser.py index be97a15..0b57321 100644 --- a/shef/shef_parser.py +++ b/shef/shef_parser.py @@ -5230,34 +5230,56 @@ def run_export( loglevel, ): """Export timeseries/group from CWMS via CDA and write to SHEF file or stdout.""" - from hec import HecTime, hectime + _relative_pattern = re.compile( + r"^\s*T\s*([+-])\s*(\d+)\s*([SMHDY])\s*$", re.IGNORECASE + ) + _relative_units = { + "S": "seconds", + "M": "minutes", + "H": "hours", + "D": "days", + "Y": "years", + } - def parse_dt(s: Optional[str]): + def parse_dt(s: Optional[str], now: Optional[datetime] = None) -> Optional[datetime]: + """Parse an ISO 8601 datetime or a HEC-style relative time (T, T-1D, T+2H, ...).""" if not s: return None + s = s.strip() + ref = now if now is not None else datetime.utcnow() + if s.upper() == "T": + return ref + m = _relative_pattern.match(s) + if m: + sign = 1 if m.group(1) == "+" else -1 + qty = int(m.group(2)) * sign + unit = m.group(3).upper() + if unit == "Y": + return ref.replace(year=ref.year + qty) + return ref + timedelta(**{_relative_units[unit]: qty}) try: - return datetime.fromisoformat(s).strftime("%m/%d/%Y %H:%M:%S") - except Exception: + return datetime.fromisoformat(s) + except ValueError: try: - return datetime.fromisoformat(s + "T00:00:00").strftime( - "%m/%d/%Y %H:%M:%S" + return datetime.fromisoformat(s + "T00:00:00") + except ValueError: + raise click.BadParameter( + f"Could not parse time [{s}]; expected ISO 8601 (e.g. 2024-01-15T06:00) or relative (e.g. T, T-1D, T+2H)" ) - except Exception: - return s if timeseries_ids is not None: ts_ids = timeseries_ids.replace(" ", "").split(",") else: ts_ids = None - start = HecTime() - end = HecTime() - st = parse_dt(start_time) - et = parse_dt(end_time) - window = st + ", " + et - if hectime.get_time_window(window, start, end) == -1: - click.BadParameter( - f"Invalid time window check start and end times entered: {window}" + now = datetime.utcnow() + start_dt = parse_dt(start_time, now=now) + end_dt = parse_dt(end_time, now=now) + if start_dt is None or end_dt is None: + raise click.BadParameter("Both --start-time and --end-time are required") + if end_dt < start_dt: + raise click.BadParameter( + f"Invalid time window: end [{end_dt}] is before start [{start_dt}]" ) # configure logging for the export command to match parse() behavior @@ -5276,8 +5298,8 @@ def parse_dt(s: Optional[str]): export_file=export_file, timeseries_group=timeseries_group, timeseries_ids=ts_ids, - start_time=start.datetime(), - end_time=end.datetime(), + start_time=start_dt, + end_time=end_dt, ) except ValueError as e: # convert validation errors to Click exceptions so CLI shows a friendly message diff --git a/tests/test_cda_exporter_empty_series.py b/tests/test_cda_exporter_empty_series.py new file mode 100644 index 0000000..ba8ce57 --- /dev/null +++ b/tests/test_cda_exporter_empty_series.py @@ -0,0 +1,127 @@ +"""Regression tests for issue #75: export must not fail when some time series in a group are empty.""" +import json +import types +from io import StringIO + +import pytest + +from shef.exporters import cda_exporter as cda_exporter_mod +from shef.loaders import cda_loader as cda_loader_mod + + +class _FakeResponse: + def __init__(self, payload): + self.json = payload + + +def _build_exporter(monkeypatch, tsids, response_for): + """Construct a CdaExporter wired to canned cwms responses.""" + monkeypatch.setattr(cda_loader_mod.cwms, "init_session", lambda **kw: None) + + # stub the group-fetching call so make_export_transforms returns our tsids + def fake_groups(**kw): + return types.SimpleNamespace( + json=[ + { + "id": "TG", + "description": "", + "assigned-time-series": [ + { + "timeseries-id": tsid, + "office-id": "OFF", + "alias-id": f"LOC{i}.HG.RZ.1:Units=ft", + } + for i, tsid in enumerate(tsids) + ], + } + ] + ) + + monkeypatch.setattr(cda_loader_mod.cwms, "get_timeseries_groups", fake_groups) + + captured_unload = {} + + def fake_get_timeseries(**kw): + return response_for(kw["ts_id"]) + + monkeypatch.setattr(cda_exporter_mod.cwms, "get_timeseries", fake_get_timeseries) + + exporter = cda_exporter_mod.CdaExporter("http://x", "OFF") + + # capture what gets fed to the loader's unload step (parsed back from JSON) + def fake_unload(): + raw = exporter._cda_loader._input.read() + captured_unload["raw"] = raw + captured_unload["parsed"] = json.loads(raw) + + exporter._cda_loader.unload = fake_unload + exporter.set_output(StringIO()) + return exporter, captured_unload + + +def test_export_skips_empty_time_series_and_still_produces_valid_json(monkeypatch): + tsids = [ + "OFF.LOC0.Flow.Inst.1Hour.0.Raw", + "OFF.LOC1.Flow.Inst.1Hour.0.Raw", + "OFF.LOC2.Flow.Inst.1Hour.0.Raw", + ] + + def response_for(tsid): + if tsid.endswith("LOC1.Flow.Inst.1Hour.0.Raw"): + return _FakeResponse( + {"name": tsid, "office-id": "OFF", "units": "ft", "values": []} + ) + return _FakeResponse( + { + "name": tsid, + "office-id": "OFF", + "units": "ft", + "values": [[0, 1.0, 0]], + } + ) + + exporter, captured = _build_exporter(monkeypatch, tsids, response_for) + exporter.export("TG") + + # JSON must be valid (regression for the column-118899 decode failure) + assert "parsed" in captured, "unload was not invoked" + payloads = captured["parsed"] + assert isinstance(payloads, list) + # the empty LOC1 series must be omitted, the other two kept + names = [p["name"] for p in payloads] + assert tsids[0] in names + assert tsids[2] in names + assert tsids[1] not in names + + +def test_export_handles_missing_values_key(monkeypatch): + tsids = ["OFF.LOC0.Flow.Inst.1Hour.0.Raw"] + + def response_for(tsid): + return _FakeResponse({"name": tsid, "office-id": "OFF"}) + + exporter, captured = _build_exporter(monkeypatch, tsids, response_for) + exporter.export("TG") + # no values means unload is never invoked, and the build did not crash + assert "parsed" not in captured + + +def test_export_handles_cwms_exception_per_series(monkeypatch): + tsids = [ + "OFF.LOC0.Flow.Inst.1Hour.0.Raw", + "OFF.LOC1.Flow.Inst.1Hour.0.Raw", + ] + + def response_for(tsid): + if "LOC0" in tsid: + raise RuntimeError("simulated CDA outage for LOC0") + return _FakeResponse( + {"name": tsid, "office-id": "OFF", "units": "ft", "values": [[0, 1.0, 0]]} + ) + + exporter, captured = _build_exporter(monkeypatch, tsids, response_for) + exporter.export("TG") + + parsed = captured["parsed"] + assert len(parsed) == 1 + assert parsed[0]["name"] == tsids[1] diff --git a/tests/test_cda_loader_make_export_transforms.py b/tests/test_cda_loader_make_export_transforms.py new file mode 100644 index 0000000..16b9965 --- /dev/null +++ b/tests/test_cda_loader_make_export_transforms.py @@ -0,0 +1,135 @@ +import types + +from shef.loaders import cda_loader + + +def _fake_groups_response(assigned): + return types.SimpleNamespace( + json=[ + { + "id": "GROUP_A", + "description": "test group", + "assigned-time-series": assigned, + } + ] + ) + + +def _make_loader(): + loader = cda_loader.CdaLoader(logger=None) + loader._office_id = "OFF" + return loader + + +def test_make_export_transforms_skips_missing_alias_and_keeps_others(monkeypatch): + """A time series missing the alias-id key must not abort the rest of the group.""" + assigned = [ + {"timeseries-id": "OFF.BadNoAlias.Flow.Inst.1Hour.0.Raw", "office-id": "OFF"}, + { + "timeseries-id": "OFF.Good1.Flow.Inst.1Hour.0.Raw", + "office-id": "OFF", + "alias-id": "GOOD1.HG.RZ.1:Units=ft", + }, + ] + monkeypatch.setattr( + cda_loader.cwms, + "get_timeseries_groups", + lambda **kw: _fake_groups_response(assigned), + ) + + loader = _make_loader() + loader.make_export_transforms() + + assert loader._export_groups["GROUP_A"]["timeseries"] == [ + "OFF.Good1.Flow.Inst.1Hour.0.Raw" + ] + assert "OFF.Good1.Flow.Inst.1Hour.0.Raw" in loader._transforms + + +def test_make_export_transforms_skips_empty_alias_and_keeps_others(monkeypatch): + """An empty alias-id string must not abort the rest of the group.""" + assigned = [ + { + "timeseries-id": "OFF.BadEmpty.Flow.Inst.1Hour.0.Raw", + "office-id": "OFF", + "alias-id": "", + }, + { + "timeseries-id": "OFF.Good2.Flow.Inst.1Hour.0.Raw", + "office-id": "OFF", + "alias-id": "GOOD2.HG.RZ.1", + }, + ] + monkeypatch.setattr( + cda_loader.cwms, + "get_timeseries_groups", + lambda **kw: _fake_groups_response(assigned), + ) + + loader = _make_loader() + loader.make_export_transforms() + + assert loader._export_groups["GROUP_A"]["timeseries"] == [ + "OFF.Good2.Flow.Inst.1Hour.0.Raw" + ] + + +def test_make_export_transforms_scopes_fetch_to_requested_group(monkeypatch): + """When a group_id is given, only that group should be fetched from cwms (no warnings about other groups).""" + calls = [] + + def fake_get_groups(**kwargs): + calls.append(kwargs) + return _fake_groups_response( + [ + { + "timeseries-id": "OFF.Only.Flow.Inst.1Hour.0.Raw", + "office-id": "OFF", + "alias-id": "ONLY.HG.RZ.1", + } + ] + ) + + monkeypatch.setattr(cda_loader.cwms, "get_timeseries_groups", fake_get_groups) + + loader = _make_loader() + loader.make_export_transforms(group_id="GROUP_A") + + assert len(calls) == 1 + assert calls[0]["timeseries_group_like"] == "^GROUP_A$" + assert calls[0]["group_office_id"] == "OFF" + assert calls[0]["office_id"] == "OFF" + assert "GROUP_A" in loader._loaded_export_group_ids + assert loader._loaded_all_export_groups is False + + # second call for same group is a no-op (no extra fetch) + loader.make_export_transforms(group_id="GROUP_A") + assert len(calls) == 1 + + +def test_make_export_transforms_skips_malformed_alias_and_keeps_others(monkeypatch): + """A malformed alias-id (raises inside make_shef_transform) must not abort the rest.""" + assigned = [ + { + "timeseries-id": "OFF.BadMalformed.Flow.Inst.1Hour.0.Raw", + "office-id": "OFF", + "alias-id": "not-a-valid-shef-alias", + }, + { + "timeseries-id": "OFF.Good3.Flow.Inst.1Hour.0.Raw", + "office-id": "OFF", + "alias-id": "GOOD3.HG.RZ.1", + }, + ] + monkeypatch.setattr( + cda_loader.cwms, + "get_timeseries_groups", + lambda **kw: _fake_groups_response(assigned), + ) + + loader = _make_loader() + loader.make_export_transforms() + + assert "OFF.Good3.Flow.Inst.1Hour.0.Raw" in loader._export_groups["GROUP_A"][ + "timeseries" + ] diff --git a/tests/test_export_time_window.py b/tests/test_export_time_window.py new file mode 100644 index 0000000..4484265 --- /dev/null +++ b/tests/test_export_time_window.py @@ -0,0 +1,150 @@ +"""Tests for the time-window parsing inside run_export (replaces former hec dependency).""" +import sys +import types +from datetime import datetime, timedelta + +import click +import pytest +from click.testing import CliRunner + + +def _install_stub_cda_exporter(monkeypatch): + captured = {} + + class MockExporter: + def __init__(self, api_root, office): + captured["api_root"] = api_root + captured["office"] = office + self.start_time = None + self.end_time = None + + def set_output(self, out): + pass + + def export(self, ts_or_group): + captured["start_time"] = self.start_time + captured["end_time"] = self.end_time + captured["target"] = ts_or_group + + fake_mod = types.SimpleNamespace(CdaExporter=MockExporter) + monkeypatch.setitem(sys.modules, "shef.exporters.cda_exporter", fake_mod) + return captured + + +def _invoke(args): + from shef.shef_parser import cli + + return CliRunner().invoke(cli, args) + + +def test_iso_start_and_end_parse_to_datetimes(monkeypatch, tmp_path): + captured = _install_stub_cda_exporter(monkeypatch) + result = _invoke( + [ + "export", + "--api-root", + "http://x", + "--office", + "OFF", + "--timeseries-group", + "TG", + "--start-time", + "2024-01-15T06:00:00", + "--end-time", + "2024-01-16T06:00:00", + "--export-file", + str(tmp_path / "out.shef"), + ] + ) + assert result.exit_code == 0, result.output + assert captured["start_time"] == datetime(2024, 1, 15, 6, 0, 0) + assert captured["end_time"] == datetime(2024, 1, 16, 6, 0, 0) + + +def test_iso_date_only_parses_as_midnight(monkeypatch, tmp_path): + captured = _install_stub_cda_exporter(monkeypatch) + result = _invoke( + [ + "export", + "--api-root", + "http://x", + "--office", + "OFF", + "--timeseries-group", + "TG", + "--start-time", + "2024-01-15", + "--end-time", + "2024-01-16", + "--export-file", + str(tmp_path / "out.shef"), + ] + ) + assert result.exit_code == 0, result.output + assert captured["start_time"] == datetime(2024, 1, 15, 0, 0, 0) + assert captured["end_time"] == datetime(2024, 1, 16, 0, 0, 0) + + +def test_relative_T_minus_1D_and_T(monkeypatch, tmp_path): + captured = _install_stub_cda_exporter(monkeypatch) + result = _invoke( + [ + "export", + "--api-root", + "http://x", + "--office", + "OFF", + "--timeseries-group", + "GRFT", + "--start-time", + "T-1D", + "--end-time", + "T", + "--export-file", + str(tmp_path / "out.shef"), + ] + ) + assert result.exit_code == 0, result.output + assert captured["end_time"] - captured["start_time"] == timedelta(days=1) + + +def test_invalid_time_string_reports_bad_parameter(monkeypatch, tmp_path): + _install_stub_cda_exporter(monkeypatch) + result = _invoke( + [ + "export", + "--api-root", + "http://x", + "--office", + "OFF", + "--timeseries-group", + "TG", + "--start-time", + "not-a-time", + "--end-time", + "T", + ] + ) + assert result.exit_code != 0 + assert "Could not parse time" in result.output or "not-a-time" in result.output + + +def test_end_before_start_reports_bad_parameter(monkeypatch, tmp_path): + _install_stub_cda_exporter(monkeypatch) + result = _invoke( + [ + "export", + "--api-root", + "http://x", + "--office", + "OFF", + "--timeseries-group", + "TG", + "--start-time", + "2024-01-16", + "--end-time", + "2024-01-15", + ] + ) + assert result.exit_code != 0 + assert "before start" in result.output or "Invalid time window" in result.output