Coverage for src/km3pipe/tests/test_dataclasses.py: 99%
871 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-08 03:14 +0000
1# Filename: test_dataclasses.py
2# pylint: disable=C0111,R0904,C0103
3# vim:set ts=4 sts=4 sw=4 et:
4"""
5...
7"""
9import numpy as np
10from numpy.testing import assert_array_equal, assert_allclose
11import pytest
13import km3pipe.extras
14from km3pipe.testing import TestCase, skip # noqa
15from km3pipe.dataclasses import (
16 Table,
17 NDArray,
18 Vec3,
19 inflate_dtype,
20 has_structured_dt,
21 is_structured,
22 DEFAULT_H5LOC,
23 DEFAULT_NAME,
24 DEFAULT_SPLIT,
25)
27__author__ = "Tamas Gal, Moritz Lotze"
28__copyright__ = "Copyright 2016, Tamas Gal and the KM3NeT collaboration."
29__credits__ = []
30__license__ = "MIT"
31__maintainer__ = "Tamas Gal, Moritz Lotze"
32__email__ = "tgal@km3net.de"
33__status__ = "Development"
36class TestDtypes(TestCase):
37 def setUp(self):
38 self.c_dt = np.dtype(
39 [
40 ("a", "<f4"),
41 ("origin", "<u4"),
42 ("pmt_id", "<u4"),
43 ("time", "<f8"),
44 ("group_id", "<u4"),
45 ]
46 )
48 def test_is_structured(self):
49 assert is_structured(self.c_dt)
50 assert not is_structured(np.dtype("int64"))
51 assert not is_structured(np.dtype(int))
52 assert not is_structured(np.dtype(float))
54 def test_has_structured_dt(self):
55 assert has_structured_dt(np.ones(2, dtype=self.c_dt))
56 assert not has_structured_dt(np.ones(2, dtype=float))
57 assert not has_structured_dt(np.ones(2, dtype=int))
58 assert not has_structured_dt([1, 2, 3])
59 assert not has_structured_dt([1.0, 2.0, 3.0])
60 assert not has_structured_dt([1.0, 2, 3.0])
61 assert not has_structured_dt([])
63 def test_inflate_hasstructured(self):
64 arr = np.ones(3, dtype=self.c_dt)
65 names = ["a", "b", "c"]
66 print(arr.dtype)
67 assert has_structured_dt(arr)
68 dt_a = inflate_dtype(arr, names=names)
69 assert dt_a == self.c_dt
71 def test_inflate_nostructured(self):
72 names = ["a", "b", "c"]
73 arr = [1, 2, 3]
74 assert not has_structured_dt(arr)
75 dt_l = inflate_dtype(arr, names=names)
76 assert dt_l == np.dtype([("a", "<i8"), ("b", "<i8"), ("c", "<i8")])
78 def test_inflate_mixed_casts_up(self):
79 arr = [1, 2, 3.0]
80 names = ["a", "b", "c"]
81 assert not has_structured_dt(arr)
82 dt_a = inflate_dtype(arr, names=names)
83 assert dt_a == np.dtype([("a", "<f8"), ("b", "<f8"), ("c", "<f8")])
86class TestTable(TestCase):
87 def setUp(self):
88 self.dt = np.dtype([("a", int), ("b", float), ("group_id", int)])
89 self.arr = np.array(
90 [
91 (0, 1.0, 2),
92 (3, 7.0, 5),
93 (6, 4.0, 8),
94 ],
95 dtype=self.dt,
96 )
98 def test_h5loc(self):
99 tab = self.arr.view(Table)
100 assert tab.h5loc == DEFAULT_H5LOC
101 tab = Table(self.arr)
102 assert tab.h5loc == DEFAULT_H5LOC
103 tab = Table(self.arr, h5loc="/foo")
104 assert tab.h5loc == "/foo"
106 def test_split(self):
107 tab = self.arr.view(Table)
108 assert tab.split_h5 is False
109 tab = Table(self.arr)
110 assert tab.split_h5 is False
111 tab = Table(self.arr, split_h5=True)
112 assert tab.split_h5
114 def test_name(self):
115 tab = self.arr.view(Table)
116 assert tab.name == DEFAULT_NAME
117 tab = Table(self.arr)
118 assert tab.name == DEFAULT_NAME
119 tab = Table(self.arr, name="foo")
120 assert tab.name == "foo"
122 def test_view(self):
123 tab = self.arr.view(Table)
124 assert tab.dtype == self.dt
125 assert tab.h5loc == DEFAULT_H5LOC
126 assert_array_equal(tab.a, np.array([0, 3, 6]))
127 assert tab[0]["group_id"] == 2
128 assert tab[0].group_id == 2
129 assert tab["group_id"][0] == 2
130 assert tab.group_id[0] == 2
131 assert isinstance(tab[0], np.record)
132 for row in tab:
133 assert isinstance(row, np.record)
134 assert row["a"] == 0
135 assert row.a == 0
136 for c in row:
137 assert c == 0
138 break
139 assert_allclose([0, 1.0, 2], [c for c in row])
140 break
142 def test_init(self):
143 tab = Table(self.arr)
144 assert tab.h5loc == DEFAULT_H5LOC
145 tab = Table(self.arr, h5loc="/bla")
146 assert tab.dtype == self.dt
147 assert tab.h5loc == "/bla"
148 assert_array_equal(tab.a, np.array([0, 3, 6]))
149 assert tab[0]["group_id"] == 2
150 assert tab[0].group_id == 2
151 assert tab["group_id"][0] == 2
152 assert tab.group_id[0] == 2
153 assert isinstance(tab[0], np.record)
154 for row in tab:
155 assert isinstance(row, np.record)
156 assert row["a"] == 0
157 assert row.a == 0
158 for c in row:
159 assert c == 0
160 break
161 assert_allclose([0, 1.0, 2], [c for c in row])
162 break
164 def test_fromdict(self):
165 n = 5
166 dmap = {
167 "a": np.ones(n, dtype=int),
168 "b": np.zeros(n, dtype=float),
169 "c": 0,
170 }
171 # tab = Table.from_dict(dmap)
172 # self.assertTrue(isinstance(tab, Table))
173 # assert tab.h5loc == DEFAULT_H5LOC
174 dt = [("a", float), ("b", float), ("c", float)]
175 tab = Table.from_dict(dmap, dtype=dt)
176 assert tab.h5loc == DEFAULT_H5LOC
177 assert isinstance(tab, Table)
178 tab = Table.from_dict(dmap, dtype=dt, h5loc="/foo")
179 assert tab.h5loc == "/foo"
180 assert isinstance(tab, Table)
181 bad_dt = [("a", float), ("b", float), ("c", float), ("d", int)]
182 with pytest.raises(KeyError):
183 tab = Table.from_dict(dmap, dtype=bad_dt)
185 def test_from_dict_without_dtype(self):
186 data = {"b": [1, 2], "c": [3, 4], "a": [5, 6]}
187 tab = Table.from_dict(data)
188 assert np.allclose([1, 2], tab.b)
189 assert np.allclose([3, 4], tab.c)
190 assert np.allclose([5, 6], tab.a)
192 def test_from_dict_with_unordered_columns_wrt_to_dtype_fields(self):
193 data = {"b": [1, 2], "c": [3, 4], "a": [5, 6]}
194 dt = [("a", float), ("b", float), ("c", float)]
195 tab = Table.from_dict(data, dtype=dt)
196 assert np.allclose([1, 2], tab.b)
197 assert np.allclose([3, 4], tab.c)
198 assert np.allclose([5, 6], tab.a)
200 def test_from_dict_with_fillna(self):
201 data = {"a": [1, 2], "b": [3, 4]}
202 dt = [("a", float), ("b", float), ("c", float)]
203 tab = Table.from_dict(data, dtype=dt, fillna=True)
204 assert np.isnan(tab.c[0])
205 assert np.isnan(tab.c[1])
207 def test_init_implicitly_from_dict_with_fillna(self):
208 data = {"a": [1, 2], "b": [3, 4]}
209 dt = [("a", float), ("b", float), ("c", float)]
210 tab = Table(data, dtype=dt, fillna=True)
211 assert np.isnan(tab.c[0])
212 assert np.isnan(tab.c[1])
214 def test_from_dict_doesnt_alter_original_dict(self):
215 a = [1, 2]
216 b = [False, True]
217 c = 3
218 data = {"a": a, "b": b, "c": c}
219 tab = Table(data)
220 assert data["a"] is a
221 assert data["b"] is b
222 assert data["c"] is c
224 def test_fromcolumns(self):
225 n = 5
226 dlist = [
227 np.ones(n, dtype=int),
228 np.zeros(n, dtype=float),
229 0,
230 ]
231 dt = np.dtype([("a", float), ("b", float), ("c", float)])
232 with pytest.raises(ValueError):
233 tab = Table(dlist, dtype=dt)
234 tab = Table.from_columns(dlist, dtype=dt)
235 print(tab.dtype)
236 print(tab.shape)
237 print(tab)
238 assert tab.h5loc == DEFAULT_H5LOC
239 assert isinstance(tab, Table)
240 tab = Table.from_columns(dlist, dtype=dt, h5loc="/foo")
241 print(tab.dtype)
242 print(tab.shape)
243 print(tab)
244 assert tab.h5loc == "/foo"
245 assert isinstance(tab, Table)
246 bad_dt = [("a", float), ("b", float), ("c", float), ("d", int)]
247 with pytest.raises(ValueError):
248 tab = Table.from_columns(dlist, dtype=bad_dt)
249 print(tab.dtype)
250 print(tab.shape)
251 print(tab)
253 def test_from_columns_with_colnames(self):
254 t = Table.from_columns(
255 [
256 [1, 2, 3],
257 [4, 5, 6],
258 [7, 8, 9],
259 [10, 11, 12],
260 [13, 14, 15],
261 [16, 17, 18],
262 [19, 20, 21],
263 ],
264 colnames=["a", "b", "c", "d", "e", "f", "g"],
265 )
266 print("t.a: {}".format(t.a))
267 assert np.allclose([1, 2, 3], t.a)
268 print("t.b: {}".format(t.b))
269 assert np.allclose([4, 5, 6], t.b)
271 def test_from_columns_with_colnames_upcasts(self):
272 t = Table.from_columns([[1, 2, 3], [4, 5.0, 6]], colnames=["a", "b"])
273 assert t.dtype == np.dtype([("a", float), ("b", float)])
275 def test_from_columns_with_mismatching_columns_and_dtypes_raises(self):
276 with pytest.raises(ValueError):
277 Table.from_columns([[1, 2, 3], [4, 5, 6]], dtype=np.dtype([("a", "f4")]))
279 def test_from_rows_with_colnames(self):
280 t = Table.from_rows([[1, 2], [3, 4], [5, 6]], colnames=["a", "b"])
281 assert t.dtype == np.dtype([("a", int), ("b", int)])
282 assert np.allclose([1, 3, 5], t.a)
283 assert np.allclose([2, 4, 6], t.b)
285 def test_from_rows_with_colnames_upcasts(self):
286 t = Table.from_rows([[1, 2], [3.0, 4], [5, 6]], colnames=["a", "b"])
287 assert t.dtype == np.dtype([("a", float), ("b", float)])
289 def test_from_rows_dim(self):
290 t = Table.from_rows([[1, 2], [3.0, 4], [5, 6]], colnames=["a", "b"])
291 assert t.shape == (3,)
293 def test_from_columns_dim(self):
294 t = Table.from_columns([[1, 2, 3], [4, 5.0, 6]], colnames=["a", "b"])
295 assert t.shape == (3,)
297 def test_fromrows(self):
298 dlist = [
299 [1, 2, 3],
300 [4, 5, 6],
301 ]
302 dt = np.dtype([("a", float), ("b", float), ("c", float)])
303 with pytest.raises(ValueError):
304 tab = Table(dlist, dtype=dt)
305 tab = Table.from_rows(dlist, dtype=dt)
306 print(tab.dtype)
307 print(tab.shape)
308 print(tab)
309 assert tab.h5loc == DEFAULT_H5LOC
310 assert isinstance(tab, Table)
311 tab = Table.from_rows(dlist, dtype=dt, h5loc="/foo")
312 print(tab.dtype)
313 print(tab.shape)
314 print(tab)
315 assert tab.h5loc == "/foo"
316 assert isinstance(tab, Table)
317 bad_dt = [("a", float), ("b", float), ("c", float), ("d", int)]
318 with pytest.raises(ValueError):
319 tab = Table.from_rows(dlist, dtype=bad_dt)
320 print(tab.dtype)
321 print(tab.shape)
322 print(tab)
324 def test_expand_scalars(self):
325 dmap = {
326 "a": 1,
327 "b": 0.0,
328 "c": 0,
329 }
330 t1 = Table._expand_scalars(dmap)
331 assert len(t1) > 0
332 dmap2 = {
333 "a": [1, 2, 1],
334 "b": 0.0,
335 "c": [0, 1],
336 }
337 t2 = Table._expand_scalars(dmap2)
338 assert len(t2) > 0
339 dmap3 = {
340 "a": [1, 2, 1],
341 "b": [0.0],
342 "c": [0, 1],
343 }
344 t3 = Table._expand_scalars(dmap3)
345 assert len(t3) > 0
346 dmap4 = {
347 "a": [1, 2, 1],
348 "b": np.array(0.0),
349 "c": [0, 1],
350 }
351 t4 = Table._expand_scalars(dmap4)
352 assert len(t4) > 0
353 dmap5 = {
354 "a": [1, 2, 1],
355 "b": np.array([1]),
356 "c": [0, 1],
357 }
358 t5 = Table._expand_scalars(dmap5)
359 assert len(t5) > 0
361 def test_from_flat_dict(self):
362 dmap = {
363 "a": 1,
364 "b": 0.0,
365 "c": 0,
366 }
367 # tab = Table.from_dict(dmap)
368 # self.assertTrue(isinstance(tab, Table))
369 # assert tab.h5loc == DEFAULT_H5LOC
370 dt = [("a", float), ("b", float), ("c", float)]
371 tab = Table.from_dict(dmap, dtype=dt)
372 assert tab.h5loc == DEFAULT_H5LOC
373 assert isinstance(tab, Table)
374 tab = Table.from_dict(dmap, dtype=dt, h5loc="/foo")
375 assert tab.h5loc == "/foo"
376 assert isinstance(tab, Table)
377 bad_dt = [("a", float), ("b", float), ("c", float), ("d", int)]
378 with pytest.raises(KeyError):
379 tab = Table.from_dict(dmap, dtype=bad_dt)
381 def test_from_mixed_dict(self):
382 dmap = {
383 "a": 1,
384 "b": 0.0,
385 "c": np.zeros(4),
386 }
387 # tab = Table.from_dict(dmap)
388 # self.assertTrue(isinstance(tab, Table))
389 # assert tab.h5loc == DEFAULT_H5LOC
390 dt = [("a", float), ("b", float), ("c", float)]
391 tab = Table.from_dict(dmap, dtype=dt)
392 assert tab.h5loc == DEFAULT_H5LOC
393 assert isinstance(tab, Table)
394 tab = Table.from_dict(dmap, dtype=dt, h5loc="/foo")
395 assert tab.h5loc == "/foo"
396 assert isinstance(tab, Table)
397 bad_dt = [("a", float), ("b", float), ("c", float), ("d", int)]
398 with pytest.raises(KeyError):
399 tab = Table.from_dict(dmap, dtype=bad_dt)
401 def test_from_2d(self):
402 l2d = [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)]
403 names = ["a", "origin", "pmt_id", "time", "group_id"]
404 dta = inflate_dtype(l2d, names)
405 with pytest.raises(ValueError):
406 t = Table(l2d)
407 with pytest.raises(ValueError):
408 t = Table(l2d, dtype=None)
409 with pytest.raises(ValueError):
410 t = Table(l2d, colnames=names)
411 with pytest.raises(ValueError):
412 t = Table(l2d, dtype=dta)
413 with pytest.raises(ValueError):
414 t = Table(l2d, dtype=dta, colnames=["a", "b", "c", "d"]) # noqa
416 def test_flat_raises(self):
417 with pytest.raises(ValueError):
418 t = Table([1, 2, 3], dtype=int).dtype
419 with pytest.raises(ValueError):
420 t = Table([1, 2, 3], dtype=float).dtype
421 with pytest.raises(ValueError):
422 t = Table([1, 2, 3], dtype=None).dtype
423 with pytest.raises(ValueError):
424 t = Table([1, 2, 3]).dtype
425 with pytest.raises(ValueError):
426 t = Table([1, 2, 3], colnames=["a", "b", "c"]) # noqa
428 def test_init_with_unstructured_raises_valueerror(self):
429 with pytest.raises(ValueError):
430 Table(np.array([[1, 2, 3], [4, 5, 6]]))
432 def test_fromdict_init(self):
433 n = 5
434 dmap = {
435 "a": np.ones(n, dtype=int),
436 "b": np.zeros(n, dtype=float),
437 "c": 0,
438 }
439 dt = [("a", float), ("b", float), ("c", float)]
440 tab = Table(dmap, dtype=dt)
441 assert tab.h5loc == DEFAULT_H5LOC
442 self.assertTrue(isinstance(tab, Table))
443 tab = Table(dmap, dtype=dt, h5loc="/foo")
444 assert tab.h5loc == "/foo"
445 self.assertTrue(isinstance(tab, Table))
446 bad_dt = [("a", float), ("b", float), ("c", float), ("d", int)]
447 with pytest.raises(KeyError):
448 tab = Table(dmap, dtype=bad_dt)
450 def test_from_record(self):
451 t = Table({"a": [0, 1, 2], "b": [10, 20, 30]})
452 t2 = Table(t[1])
453 assert 1 == t2[0].a
454 assert 1 == t2.a[0]
455 assert 20 == t2[0].b
456 assert 20 == t2.b[0]
458 def test_append_columns(self):
459 tab = Table(self.arr)
460 print(tab)
461 with pytest.raises(ValueError):
462 tab = tab.append_columns("new", [1, 2, 3, 4])
463 tab = tab.append_columns("new", [1, 2, 3])
464 print(tab)
465 assert tab.new[0] == 1
466 assert tab.new[-1] == 3
467 tab = tab.append_columns("bar", 0)
468 print(tab)
469 assert tab.bar[0] == 0
470 assert tab.bar[-1] == 0
471 tab = tab.append_columns("lala", [1])
472 print(tab)
473 assert tab.lala[0] == 1
474 assert tab.lala[-1] == 1
475 with pytest.raises(ValueError):
476 tab = tab.append_columns(["m", "n"], [1, 2])
477 with pytest.raises(ValueError):
478 tab = tab.append_columns(["m", "n"], [[1], [2]])
479 tab = tab.append_columns(["m", "n"], [[1, 1, 2], [2, 4, 5]])
480 print(tab)
481 assert tab.m[0] == 1
482 assert tab.m[-1] == 2
483 assert tab.n[0] == 2
484 assert tab.n[-1] == 5
486 def test_append__single_column(self):
487 tab = Table({"a": 1})
488 print(tab.dtype)
489 tab = tab.append_columns(["b"], np.array([[2]]))
490 print(tab.dtype)
491 print(tab.b)
493 def test_append_columns_with_single_value(self):
494 tab = Table({"a": 1})
495 tab = tab.append_columns("group_id", 0)
496 assert 0 == tab.group_id[0]
498 def test_append_columns_with_multiple_values(self):
499 tab = Table({"a": [1, 2]})
500 tab = tab.append_columns("group_id", [0, 1])
501 assert 0 == tab.group_id[0]
502 assert 1 == tab.group_id[1]
504 def test_append_columns_modifies_dtype(self):
505 tab = Table({"a": [1, 2]})
506 tab = tab.append_columns("group_id", [0, 1])
507 assert "group_id" in tab.dtype.names
509 def test_append_column_which_is_too_short_raises(self):
510 tab = Table({"a": [1, 2, 3]})
511 with pytest.raises(ValueError):
512 tab = tab.append_columns("b", [4, 5])
514 def test_append_columns_duplicate(self):
515 tab = Table({"a": 1})
516 with pytest.raises(ValueError):
517 tab = tab.append_columns(["a"], np.array([[2]]))
519 def test_append_columns_with_mismatching_lengths_raises(self):
520 tab = Table({"a": [1, 2, 3]})
521 with pytest.raises(ValueError):
522 tab.append_columns(colnames=["b", "c"], values=[[4, 5, 6], [7, 8]])
524 def test_append_columns_which_is_too_long(self):
525 tab = Table({"a": [1, 2, 3]})
526 with pytest.raises(ValueError):
527 tab.append_columns("b", values=[4, 5, 6, 7])
529 def test_drop_column(self):
530 tab = Table({"a": 1, "b": 2})
531 tab = tab.drop_columns("a")
532 with pytest.raises(AttributeError):
533 print(tab.a)
534 print(tab.b)
536 def test_drop_columns(self):
537 tab = Table({"a": 1, "b": 2, "c": 3})
538 print(tab.dtype)
539 tab = tab.drop_columns(["a", "b"])
540 print(tab.dtype)
541 with pytest.raises(AttributeError):
542 print(tab.a)
543 with pytest.raises(AttributeError):
544 print(tab.b)
545 print(tab.c)
547 def test_template(self):
548 n = 10
549 channel_ids = np.arange(n)
550 dom_ids = np.arange(n)
551 times = np.arange(n)
552 tots = np.arange(n)
553 triggereds = np.ones(n)
554 d_hits = {
555 "channel_id": channel_ids,
556 "dom_id": dom_ids,
557 "time": times,
558 "tot": tots,
559 "triggered": triggereds,
560 "group_id": 0, # event_id
561 }
562 tab = Table.from_template(d_hits, "Hits")
563 assert tab.name == "Hits"
564 assert tab.split_h5 is True
565 assert isinstance(tab, Table)
566 ar_hits = {
567 "channel_id": np.ones(n, dtype=int),
568 "dom_id": np.ones(n, dtype=int),
569 "time": np.ones(n, dtype=float),
570 "tot": np.ones(n, dtype=float),
571 "triggered": np.ones(n, dtype=bool),
572 "group_id": np.ones(n, dtype=int),
573 }
574 tab = Table.from_template(ar_hits, "Hits")
575 assert tab.name == "Hits"
576 assert tab.split_h5 is True
577 assert isinstance(tab, Table)
579 def test_incomplete_template(self):
580 n = 10
581 channel_ids = np.arange(n)
582 dom_ids = np.arange(n)
583 # times = np.arange(n)
584 tots = np.arange(n)
585 triggereds = np.ones(n)
586 d_hits = {
587 "channel_id": channel_ids,
588 "dom_id": dom_ids,
589 # 'time': times,
590 "tot": tots,
591 "triggered": triggereds,
592 "group_id": 0, # event_id
593 }
594 with pytest.raises(KeyError):
595 tab = Table.from_template(d_hits, "Hits")
596 assert tab is not None
597 ar_hits = {
598 "channel_id": np.ones(n, dtype=int),
599 "dom_id": np.ones(n, dtype=int),
600 # 'time': np.ones(n, dtype=float),
601 "tot": np.ones(n, dtype=float),
602 "triggered": np.ones(n, dtype=bool),
603 "group_id": np.ones(n, dtype=int),
604 }
605 with pytest.raises(KeyError):
606 tab = Table.from_template(ar_hits, "Hits")
607 assert tab is not None
609 def test_adhoc_template(self):
610 a_template = {
611 "dtype": np.dtype([("a", "<u4"), ("b", "f4")]),
612 "h5loc": "/yat",
613 "split_h5": True,
614 "h5singleton": True,
615 "name": "YetAnotherTemplate",
616 }
617 arr = np.array([(1, 3), (2, 4)], dtype=a_template["dtype"])
618 tab = Table.from_template(arr, a_template)
619 self.assertListEqual([1, 2], list(tab.a))
620 self.assertListEqual([3.0, 4.0], list(tab.b))
621 assert "YetAnotherTemplate" == tab.name
622 assert tab.h5singleton
624 def test_adhoc_noname_template(self):
625 a_template = {
626 "dtype": np.dtype([("a", "<u4"), ("b", "f4")]),
627 "h5loc": "/yat",
628 "split_h5": True,
629 "h5singleton": True,
630 }
631 arr = np.array([(1, 3), (2, 4)], dtype=a_template["dtype"])
632 tab = Table.from_template(arr, a_template)
633 self.assertListEqual([1, 2], list(tab.a))
634 self.assertListEqual([3.0, 4.0], list(tab.b))
635 assert DEFAULT_NAME == tab.name
636 assert tab.h5singleton
638 def test_element_list_with_dtype(self):
639 bad_elist = [
640 [1, 2.1],
641 [3, 4.2],
642 [5, 6.3],
643 ]
644 dt = np.dtype([("a", int), ("b", float)])
645 print("list(list)")
646 arr_bad = np.array(bad_elist, dtype=dt)
647 print(arr_bad)
648 elist = [tuple(el) for el in bad_elist]
649 arr = np.array(elist, dtype=dt)
650 print("list(tuple)")
651 print(arr)
652 tab = Table(arr)
653 print(tab)
654 assert tab.a[0] == 1
656 def test_sort(self):
657 dt = np.dtype([("a", int), ("b", float), ("c", int)])
658 arr = np.array(
659 [
660 (0, 1.0, 2),
661 (3, 7.0, 5),
662 (6, 4.0, 8),
663 ],
664 dtype=dt,
665 )
666 tab = Table(arr)
667 tab_sort = tab.sorted("b")
668 assert_array_equal(tab_sort["a"], np.array([0, 6, 3]))
670 def test_init_directly_with_df(self):
671 pd = km3pipe.extras.pandas()
673 df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
674 tab = Table(df, h5loc="/foo")
675 assert np.allclose(df.a, tab.a)
676 assert np.allclose(df.b, tab.b)
677 assert tab.h5loc == "/foo"
679 def test_df(self):
680 pd = km3pipe.extras.pandas()
682 dt = np.dtype([("a", int), ("b", float), ("c", int)])
683 arr = np.array(
684 [
685 (0, 1.0, 2),
686 (3, 7.0, 5),
687 (6, 4.0, 8),
688 ],
689 dtype=dt,
690 )
691 print(dir(Table))
692 df = pd.DataFrame(arr)
693 tab = Table.from_dataframe(df, h5loc="/bla")
694 df2 = tab.to_dataframe()
695 pd.testing.assert_frame_equal(df, df2)
697 def test_slicing(self):
698 dt = np.dtype([("a", int), ("b", float), ("c", bool)])
699 arr = np.array(
700 [
701 (0, 1.0, True),
702 (2, 3.0, False),
703 (4, 5.0, True),
704 ],
705 dtype=dt,
706 )
707 tab = Table(arr)
708 assert 2 == len(tab[tab.c])
709 assert 1 == len(tab[tab.b > 3.0])
711 def test_contains(self):
712 dt = np.dtype([("a", int), ("b", float), ("c", bool)])
713 arr = np.array(
714 [
715 (0, 1.0, True),
716 (2, 3.0, False),
717 (4, 5.0, True),
718 ],
719 dtype=dt,
720 )
721 tab = Table(arr)
722 assert "a" in tab
723 assert "b" in tab
724 assert "c" in tab
725 assert "d" not in tab
727 def test_index_returns_reference(self):
728 tab = Table({"a": [1, 2, 3]})
729 tab[1].a = 4
730 assert np.allclose(tab.a, [1, 4, 3])
732 def test_index_of_attribute_returns_reference(self):
733 tab = Table({"a": [1, 2, 3]})
734 tab.a[1] = 4
735 assert np.allclose(tab.a, [1, 4, 3])
737 def test_mask_returns_copy(self):
738 tab = Table({"a": [1, 2, 3]})
739 tab[[True, False, True]].a = [4, 5]
740 assert np.allclose(tab.a, [1, 2, 3])
742 def test_mask_on_attribute_returns_reference(self):
743 tab = Table({"a": [1, 2, 3]})
744 tab.a[[True, False, True]] = [4, 5]
745 assert np.allclose(tab.a, [4, 2, 5])
747 def test_index_mask_returns_copy(self):
748 tab = Table({"a": [1, 2, 3]})
749 tab[[1, 2]].a = [4, 5]
750 assert np.allclose(tab.a, [1, 2, 3])
752 def test_index_mask_of_attribute_returns_reference(self):
753 tab = Table({"a": [1, 2, 3]})
754 tab.a[[1, 2]] = [4, 5]
755 assert np.allclose(tab.a, [1, 4, 5])
757 def test_slice_returns_reference(self):
758 tab = Table({"a": [1, 2, 3]})
759 tab[:2].a = [4, 5]
760 assert np.allclose(tab.a, [4, 5, 3])
762 def test_slice_of_attribute_returns_reference(self):
763 tab = Table({"a": [1, 2, 3]})
764 tab.a[:2] = [4, 5]
765 assert np.allclose(tab.a, [4, 5, 3])
767 def test_slice_keeps_metadata(self):
768 tab = Table(
769 {"a": [1, 2, 3]},
770 h5loc="/lala",
771 split_h5=True,
772 name="bla",
773 h5singleton=False,
774 )
775 assert tab[:2].h5loc == "/lala"
776 assert tab[:2].name == "bla"
777 assert not tab[:2].h5singleton
778 assert tab[:2].split_h5
780 def test_mask_keeps_metadata(self):
781 tab = Table(
782 {"a": [1, 2, 3]}, h5loc="/lala", split_h5=True, name="bla", h5singleton=True
783 )
784 m = np.ones(len(tab), dtype=bool)
785 assert tab[m].h5loc == "/lala"
786 assert tab[m].name == "bla"
787 assert tab[m].h5singleton
788 assert tab[m].split_h5
790 def test_indexing_keeps_metadata(self):
791 tab = Table(
792 {"a": [1, 2, 3]}, h5loc="/lala", split_h5=True, name="bla", h5singleton=True
793 )
794 im = [1, 1, 0]
795 assert tab[im].h5loc == "/lala"
796 assert tab[im].name == "bla"
797 assert tab[im].h5singleton
798 assert tab[im].split_h5
800 def test_crash_repr(self):
801 a = np.array("", dtype=[("a", "<U1")])
802 with pytest.raises(TypeError):
803 print(len(a))
804 tab = Table(a)
805 s = tab.__str__()
806 assert s is not None
807 r = tab.__repr__()
808 assert r is not None
810 def test_array_finalize_with_obj_none(self):
811 tab = Table({"a": [1, 2, 3]})
812 assert tab.__array_finalize__(None) is None
814 def test_array_wrap(self):
815 t = Table({"a": [1, 2, 3], "b": [4, 5, 6]})
816 wrapped = t.__array_wrap__(np.array((Table({"a": 1}))))
817 assert wrapped.a[0] == 1
819 def test_templates_avail(self):
820 t = Table({"a": 1})
821 templates = t.templates_avail
822 assert templates
824 def test_add_table_to_itself(self):
825 tab = Table({"a": [1]})
826 added_tab = tab + tab
827 assert 2 == len(added_tab)
828 self.assertListEqual([1, 1], list(added_tab.a))
830 def test_add_two_tables(self):
831 tab1 = Table({"a": [1, 2]})
832 tab2 = Table({"a": [3, 4]})
833 added_tab = tab1 + tab2
834 assert 4 == len(added_tab)
835 self.assertListEqual([1, 2, 3, 4], list(added_tab.a))
837 def test_add_two_tables_with_different_lengths(self):
838 tab1 = Table({"a": [1, 2]})
839 tab2 = Table({"a": [3, 4, 5]})
840 added_tab = tab1 + tab2
841 assert 5 == len(added_tab)
842 self.assertListEqual([1, 2, 3, 4, 5], list(added_tab.a))
844 def test_add_two_tables_with_different_lengths_and_columns(self):
845 tab1 = Table({"a": [1, 2], "b": [100, 200]})
846 tab2 = Table({"a": [3, 4, 5], "b": [300, 400, 500]})
847 added_tab = tab1 + tab2
848 assert 5 == len(added_tab)
849 self.assertListEqual([1, 2, 3, 4, 5], list(added_tab.a))
850 self.assertListEqual([100, 200, 300, 400, 500], list(added_tab.b))
852 def test_adding_preserves_metadata(self):
853 tab1 = Table(
854 {"a": [1, 2]}, h5loc="/a", h5singleton=True, split_h5=True, name="FooTable"
855 )
856 tab2 = Table({"a": [3, 4, 5]})
857 added_tab = tab1 + tab2
858 assert "/a" == tab1.h5loc
859 assert added_tab.h5singleton
860 assert added_tab.split_h5
861 assert "FooTable" == added_tab.name
863 def test_add_tables_with_same_colnames_but_different_dtype_order(self):
864 cols1 = ("b", "a")
865 tab1 = Table.from_columns([[100, 200], [1, 2]], colnames=cols1)
866 self.assertTupleEqual(cols1, tab1.dtype.names)
867 cols2 = ("a", "b")
868 tab2 = Table.from_columns([[3, 4, 5], [300, 400, 500]], colnames=cols2)
869 added_tab = tab1 + tab2
870 self.assertListEqual([1, 2, 3, 4, 5], list(added_tab.a))
871 self.assertListEqual([100, 200, 300, 400, 500], list(added_tab.b))
872 self.assertListEqual(list(added_tab.dtype.names), list(tab1.dtype.names))
874 def test_add_table_with_different_cols(self):
875 tab1 = Table({"a": [1]})
876 tab2 = Table({"b": [2]})
877 with self.assertRaises(TypeError):
878 added_tab = tab1 + tab2
880 def test_merge(self):
881 tab1 = Table({"a": [1]}, h5loc="/a", h5singleton=True)
882 tab2 = Table({"a": [2]})
883 tab3 = Table({"a": [3]})
884 merged_tab = Table.merge([tab1, tab2, tab3])
885 assert 3 == len(merged_tab)
886 self.assertListEqual([1, 2, 3], list(merged_tab.a))
887 assert "/a" == merged_tab.h5loc
888 assert merged_tab.h5singleton
890 def test_merge_different_columns_with_no_nan_compatible_dtype(self):
891 tab1 = Table({"a": [1]}, h5loc="/a", h5singleton=True)
892 tab2 = Table({"b": [2]})
893 tab3 = Table({"c": [3]})
894 with self.assertRaises(ValueError):
895 merged_tab = Table.merge([tab1, tab2, tab3])
897 def test_merge_different_columns_with_no_nan_compatible_dtype_even_if_fillna(self):
898 tab1 = Table({"a": [1]}, h5loc="/a", h5singleton=True)
899 tab2 = Table({"b": [2]})
900 tab3 = Table({"c": [3]})
901 with self.assertRaises(ValueError):
902 merged_tab = Table.merge([tab1, tab2, tab3], fillna=True)
904 def test_merge_different_columns_fills_nan_when_fillna(self):
905 tab1 = Table({"a": [1.1]}, h5loc="/a", h5singleton=True, split_h5=True)
906 tab2 = Table({"b": [2.2]})
907 tab3 = Table({"c": [3.3]})
908 merged_tab = Table.merge([tab1, tab2, tab3], fillna=True)
909 assert 3 == len(merged_tab)
911 assert 1.1 == merged_tab.a[0]
912 assert np.isnan(merged_tab.a[1])
913 assert np.isnan(merged_tab.a[2])
915 assert np.isnan(merged_tab.b[0])
916 assert 2.2 == merged_tab.b[1]
917 assert np.isnan(merged_tab.b[2])
919 assert np.isnan(merged_tab.c[0])
920 assert np.isnan(merged_tab.c[1])
921 assert 3.3 == merged_tab.c[2]
923 assert "/a" == merged_tab.h5loc
924 assert merged_tab.h5singleton
925 assert merged_tab.split_h5
927 def test_merge_other_different_columns_fills_nan_when_fillna(self):
928 tab1 = Table({"a": [1.1, 1.2], "b": [10.1, 10.2]})
929 tab2 = Table({"a": [2.1, 2.2], "c": [100.1, 100.2]})
931 merged_tab = Table.merge([tab1, tab2], fillna=True)
933 assert 4 == len(merged_tab)
935 self.assertListEqual([1.1, 1.2, 2.1, 2.2], list(merged_tab.a))
936 self.assertListEqual([10.1, 10.2], list(merged_tab.b[:2]))
937 self.assertListEqual([100.1, 100.2], list(merged_tab.c[2:]))
938 assert np.isnan(merged_tab.c[0])
939 assert np.isnan(merged_tab.c[1])
940 assert np.isnan(merged_tab.b[2])
941 assert np.isnan(merged_tab.b[3])
943 def test_init_with_different_dicts_but_same_content(self):
944 t1 = Table({"a": [1, 2], "b": [3, 4], "c": [5, 6]})
945 t2 = Table({"c": [1, 2], "a": [3, 4], "b": [5, 6]})
947 assert t1.dtype == t2.dtype
949 def test_init_from_template_with_differently_ordered_dicts(self):
950 t1 = Table.from_template(
951 {
952 "frame_index": 1,
953 "slice_id": 2,
954 "timestamp": 3,
955 "nanoseconds": 4,
956 "n_frames": 5,
957 },
958 "TimesliceInfo",
959 )
960 t2 = Table.from_template(
961 {
962 "n_frames": 5,
963 "timestamp": 3,
964 "nanoseconds": 4,
965 "slice_id": 2,
966 "frame_index": 1,
967 },
968 "TimesliceInfo",
969 )
970 assert t1.dtype == t2.dtype
971 assert t1.frame_index[0] == t2.frame_index[0]
972 assert t1.slice_id[0] == t2.slice_id[0]
973 assert t1.nanoseconds[0] == t2.nanoseconds[0]
974 assert t1.n_frames[0] == t2.n_frames[0]
975 assert t1.timestamp[0] == t2.timestamp[0]
977 def test_reorder_dtypes(self):
978 dtype = np.dtype([("a", "<i8"), ("c", "<i8"), ("b", "<f8")])
979 dtype_reordered = np.dtype([("b", "<f8"), ("c", "<i8"), ("a", "<i8")])
981 tab = Table({"a": 1, "b": 2.5, "c": 3}, dtype=dtype)
982 assert tab.dtype == dtype
983 assert 1 == tab.a[0]
984 assert 2.5 == tab.b[0]
985 assert 3 == tab.c[0]
987 tab_reordered = Table(tab, dtype=dtype_reordered)
988 assert tab_reordered.dtype == dtype_reordered
989 assert 1 == tab_reordered.a[0]
990 assert 2.5 == tab_reordered.b[0]
991 assert 3 == tab_reordered.c[0]
993 def test_reorder_dtypes_with_differing_names_raises(self):
994 dtype = np.dtype([("a", "<i8"), ("c", "<i8"), ("b", "<f8")])
995 dtype_reordered = np.dtype([("b", "<f8"), ("a", "<i8")])
996 tab = Table({"a": 1, "b": 2.5, "c": 3}, dtype=dtype)
998 with self.assertRaises(ValueError):
999 tab2 = Table(tab, dtype=dtype_reordered)
1001 def test_reorder_dtypes_w_matching_names_but_different_types_raise(self):
1002 dtype = np.dtype([("a", "<i8"), ("c", "<i8"), ("b", "<f8")])
1003 dtype_reordered = np.dtype([("a", "<f8"), ("c", "<i8"), ("b", "<f8")])
1004 tab = Table({"a": 1, "b": 2.5, "c": 3}, dtype=dtype)
1006 with self.assertRaises(ValueError):
1007 tab2 = Table(tab, dtype=dtype_reordered)
1010class TestTableFancaAttributes(TestCase):
1011 def setUp(self):
1012 self.arr_bare = Table(
1013 {
1014 "a": [1, 2, 3],
1015 "b": [3, 4, 5],
1016 }
1017 )
1018 self.arr_wpos = Table(
1019 {
1020 "a": [1, 2, 3],
1021 "b": [3, 4, 5],
1022 "pos_x": [10, 20, 30],
1023 "pos_y": [40, 50, 60],
1024 "pos_z": [70, 80, 90],
1025 "dir_x": [10.0, 20.0, 30.0],
1026 "dir_y": [40.0, 50.0, 60.0],
1027 "dir_z": [70.0, 80.0, 90.0],
1028 }
1029 )
1031 def test_pos_getter(self):
1032 tab = Table({"pos_x": [1, 2, 3], "pos_y": [4, 5, 6], "pos_z": [7, 8, 9]})
1033 assert np.allclose([[1, 4, 7], [2, 5, 8], [3, 6, 9]], tab.pos)
1035 def test_pos_getter_for_single_entry(self):
1036 tab = Table({"pos_x": [1, 2, 3], "pos_y": [4, 5, 6], "pos_z": [7, 8, 9]})
1037 assert np.allclose([[2, 5, 8]], tab.pos[1])
1039 def test_dir_getter(self):
1040 tab = Table({"dir_x": [1, 2, 3], "dir_y": [4, 5, 6], "dir_z": [7, 8, 9]})
1041 assert np.allclose([[1, 4, 7], [2, 5, 8], [3, 6, 9]], tab.dir)
1043 def test_dir_getter_for_single_entry(self):
1044 tab = Table({"dir_x": [1, 2, 3], "dir_y": [4, 5, 6], "dir_z": [7, 8, 9]})
1045 assert np.allclose([[2, 5, 8]], tab.dir[1])
1047 def test_dir_setter(self):
1048 tab = Table({"dir_x": [1, 0, 0], "dir_y": [0, 1, 0], "dir_z": [0, 0, 1]})
1049 new_dir = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
1050 tab.dir = new_dir
1051 assert np.allclose(new_dir, tab.dir)
1053 def test_pos_setter(self):
1054 tab = Table({"pos_x": [1, 0, 0], "pos_y": [0, 1, 0], "pos_z": [0, 0, 1]})
1055 new_pos = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
1056 tab.pos = new_pos
1057 assert np.allclose(new_pos, tab.pos)
1059 def test_phi(self):
1060 tab = Table({"dir_x": [1, 0, 0], "dir_y": [0, 1, 0], "dir_z": [0, 0, 1]})
1061 p = tab.phi
1062 assert p is not None
1064 def test_phi(self):
1065 tab = Table({"dir_x": [1, 0, 0], "dir_y": [0, 1, 0], "dir_z": [0, 0, 1]})
1066 p = tab.theta
1067 assert p is not None
1069 def test_zen(self):
1070 tab = Table({"dir_x": [1, 0, 0], "dir_y": [0, 1, 0], "dir_z": [0, 0, 1]})
1071 p = tab.zenith
1072 assert p is not None
1074 def test_azi(self):
1075 tab = Table({"dir_x": [1, 0, 0], "dir_y": [0, 1, 0], "dir_z": [0, 0, 1]})
1076 p = tab.azimuth
1077 assert p is not None
1079 def test_triggered_keeps_attrs(self):
1080 n = 5
1081 channel_ids = np.arange(n)
1082 dom_ids = np.arange(n)
1083 times = np.arange(n)
1084 tots = np.arange(n)
1085 triggereds = np.array([0, 1, 1, 0, 1])
1086 hits = Table(
1087 {
1088 "channel_id": channel_ids,
1089 "dom_id": dom_ids,
1090 "time": times,
1091 "tot": tots,
1092 "triggered": triggereds,
1093 "group_id": 0, # event_id
1094 },
1095 name="hits",
1096 h5loc="/foo",
1097 split_h5=True,
1098 )
1099 triggered_hits = hits.triggered_rows
1100 assert len(triggered_hits) == 3
1101 assert triggered_hits.split_h5
1102 assert triggered_hits.name == "hits"
1103 assert triggered_hits.h5loc == "/foo"
1105 def test_triggered_missing_col_raises(self):
1106 n = 5
1107 channel_ids = np.arange(n)
1108 dom_ids = np.arange(n)
1109 times = np.arange(n)
1110 tots = np.arange(n)
1111 hits = Table(
1112 {
1113 "channel_id": channel_ids,
1114 "dom_id": dom_ids,
1115 "time": times,
1116 "tot": tots,
1117 "group_id": 0, # event_id
1118 },
1119 name="hits",
1120 h5loc="/foo",
1121 split_h5=True,
1122 )
1123 with pytest.raises(KeyError):
1124 triggered_hits = hits.triggered_rows
1125 assert triggered_hits is not None
1128class TestNDArray(TestCase):
1129 def test_init(self):
1130 arr = np.random.random((2, 3, 4))
1131 ndarr = NDArray(arr)
1133 def test_init_array(self):
1134 arr = np.random.random((2, 3, 4))
1135 arr = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
1136 ndarr = NDArray(arr)
1137 assert 1 == ndarr[0, 0, 0]
1138 assert 6 == ndarr[1, 0, 1]
1139 assert 7 == ndarr[1, 1, 0]
1140 assert "/misc" == ndarr.h5loc
1141 assert "Unnamed NDArray" == ndarr.title
1142 assert ndarr.group_id is None
1144 def test_attributes(self):
1145 ndarr = NDArray([1], h5loc="/foo", title="Foo", group_id=23)
1146 assert "/foo" == ndarr.h5loc
1147 assert "Foo" == ndarr.title
1148 assert 23 == ndarr.group_id
1150 def test_slicing_preserves_attribute(self):
1151 ndarr = NDArray([1, 2, 3], h5loc="/foo", title="Foo", group_id=23)
1152 a = ndarr[:1]
1153 assert "/foo" == a.h5loc
1154 assert "Foo" == a.title
1155 assert 23 == a.group_id
1158class TestVec3(TestCase):
1159 def test_init(self):
1160 Vec3(0, 1, 2)
1162 def test_indices(self):
1163 v = Vec3(0, 1, 2)
1164 assert 0 == v[0]
1165 assert 1 == v[1]
1166 assert 2 == v[2]
1168 def test_attributes(self):
1169 v = Vec3(0, 1, 2)
1170 assert 0 == v.x
1171 assert 1 == v.y
1172 assert 2 == v.z
1174 def test_mutability(self):
1175 v = Vec3(0, 1, 2)
1176 v.x = 42
1177 assert 42 == v.x
1179 def test_vector_addition(self):
1180 u = Vec3(1, 2, 3)
1181 v = Vec3(4, 5, 6)
1182 w = u + v
1183 assert 5 == w.x
1184 assert 7 == w.y
1185 assert 9 == w.z
1187 def test_vector_subtraction(self):
1188 u = Vec3(1, 2, 3)
1189 v = Vec3(4, 7, 11)
1190 w = u - v
1191 assert -3 == w.x
1192 assert -5 == w.y
1193 assert -8 == w.z
1195 def test_scalar_addition(self):
1196 u = Vec3(1, 2, 3)
1198 v = u + 1
1199 assert 2 == v.x
1200 assert 3 == v.y
1201 assert 4 == v.z
1202 w = 1 + u
1203 assert 2 == w.x
1204 assert 3 == w.y
1205 assert 4 == w.z
1207 def test_scalar_subtraction(self):
1208 u = Vec3(1, 2, 3)
1210 v = u - 1
1211 assert 0 == v.x
1212 assert 1 == v.y
1213 assert 2 == v.z
1214 w = 1 - u
1215 assert 0 == w.x
1216 assert -1 == w.y
1217 assert -2 == w.z
1219 def test_multiplication(self):
1220 u = Vec3(1, 2, 3)
1221 v = u * 2
1222 assert 2 == v.x
1223 assert 4 == v.y
1224 assert 6 == v.z
1225 w = 2 * u
1226 assert 2 == w.x
1227 assert 4 == w.y
1228 assert 6 == w.z
1230 def test_division(self):
1231 u = Vec3(2, 4, 8)
1232 v = u / 2
1233 assert 1 == v.x
1234 assert 2 == v.y
1235 assert 4 == v.z
1237 def test_linalg_norm(self):
1238 u = Vec3(1, 0, 0)
1239 norm = np.linalg.norm(u)
1240 assert 1 == norm
1241 u = Vec3(0, 3, 0)
1242 norm = np.linalg.norm(u)
1243 assert 3 == norm
1244 u = Vec3(1, 1, 0)
1245 norm = np.linalg.norm(u)
1246 assert np.sqrt(2) == norm