tfile.py - pism - [fork] customized build of PISM, the parallel ice sheet model (tillflux branch)
(HTM) git clone git://src.adamsgaard.dk/pism
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) LICENSE
---
tfile.py (23540B)
---
1 import PISM
2 import PISM.testing
3 import os
4 from unittest import TestCase, SkipTest
5
6 # always available
7 backends = [PISM.PISM_NETCDF3]
8
9 if PISM.Pism_USE_PARALLEL_NETCDF4:
10 backends += [PISM.PISM_NETCDF4_PARALLEL]
11
12 if PISM.Pism_USE_PNETCDF:
13 backends += [PISM.PISM_PNETCDF]
14
15 if PISM.Pism_USE_PIO:
16 # assume that ParallelIO was built with all possible libraries
17 backends += [PISM.PISM_PIO_PNETCDF, PISM.PISM_PIO_NETCDF,
18 PISM.PISM_PIO_NETCDF4C, PISM.PISM_PIO_NETCDF4P]
19
20 ctx = PISM.Context().ctx
21
22 backend_names = {PISM.PISM_NETCDF3 : "netcdf3",
23 PISM.PISM_NETCDF4_PARALLEL : "netcdf4_parallel",
24 PISM.PISM_PNETCDF : "pnetcdf",
25 PISM.PISM_PIO_NETCDF : "pio_netcdf",
26 PISM.PISM_PIO_NETCDF4P : "pio_netcdf4p",
27 PISM.PISM_PIO_NETCDF4C : "pio_netcdf4c",
28 PISM.PISM_PIO_PNETCDF: "pio_pnetcdf"}
29
30 def fail(backend):
31 assert False, "test failed (backend = {})".format(backend_names[backend])
32
33 def test_string_to_backend():
34 "PISM.string_to_backend()"
35
36 for backend, name in backend_names.items():
37 assert PISM.string_to_backend(name) == backend
38
39 try:
40 PISM.string_to_backend("invalid")
41 return False
42 except RuntimeError:
43 pass
44
45 class File(TestCase):
46
47 def test_empty_filename(self):
48 for backend in backends:
49 try:
50 f = PISM.File(ctx.com(), "", backend, PISM.PISM_READONLY,
51 ctx.pio_iosys_id())
52 fail(backend)
53 except RuntimeError:
54 pass
55
56 def test_missing_file(self):
57 for backend in backends:
58 try:
59 f = PISM.File(ctx.com(), "missing_file.nc", backend, PISM.PISM_READONLY,
60 ctx.pio_iosys_id())
61 fail(backend)
62 except RuntimeError:
63 pass
64
65 def test_backend_guessing(self):
66 "File(..., PISM_GUESS, ...)"
67
68 f = PISM.File(ctx.com(), self.file_with_time, PISM.PISM_GUESS, PISM.PISM_READONLY,
69 ctx.pio_iosys_id())
70 assert f.nrecords() == 1
71
72 def test_create_clobber(self):
73 "File(..., PISM_READWRITE_CLOBBER)"
74 try:
75 for backend in backends:
76 f = PISM.File(ctx.com(), "test_filename.nc", backend, PISM.PISM_READWRITE_CLOBBER,
77 ctx.pio_iosys_id())
78 finally:
79 os.remove("test_filename.nc")
80
81 def test_create_move(self):
82 "File(..., PISM_READWRITE_MOVE)"
83 try:
84 for backend in backends:
85 f = PISM.File(ctx.com(), "test_filename.nc", backend, PISM.PISM_READWRITE_MOVE,
86 ctx.pio_iosys_id())
87 finally:
88 os.remove("test_filename.nc")
89 try:
90 os.remove("test_filename.nc~")
91 except:
92 pass
93
94 def test_backend(self):
95 "File.backend()"
96 for backend in backends:
97 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
98 ctx.pio_iosys_id())
99 assert f.backend() == backend
100 f.close()
101
102 def test_com(self):
103 "File.com()"
104 for backend in backends:
105 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
106 ctx.pio_iosys_id())
107 assert f.com() == ctx.com()
108 f.close()
109
110 def test_close(self):
111 "File.close()"
112 for backend in backends:
113 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
114 ctx.pio_iosys_id())
115 f.close()
116
117 try:
118 f.close()
119 # closing twice is an error
120 fail(backend)
121 except RuntimeError:
122 pass
123
124 def test_redef(self):
125 "File.redef()"
126 for backend in backends:
127 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE,
128 ctx.pio_iosys_id())
129 f.redef()
130 f.close()
131
132 def test_enddef(self):
133 "File.enddef()"
134 for backend in backends:
135 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE,
136 ctx.pio_iosys_id())
137 f.enddef()
138 f.close()
139
140 def test_sync(self):
141 "File.sync()"
142 for backend in backends:
143 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE,
144 ctx.pio_iosys_id())
145 f.sync()
146 f.close()
147
148 def test_filename(self):
149 "File.filename()"
150 for backend in backends:
151 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
152 ctx.pio_iosys_id())
153 assert f.filename() == self.file_with_time
154 f.close()
155
156 def test_nrecords(self):
157 "File.nrecords()"
158 for F in self.files:
159 for backend in backends:
160 f = PISM.File(ctx.com(), F, backend, PISM.PISM_READONLY,
161 ctx.pio_iosys_id())
162 assert f.nrecords() == 1
163 f.close()
164
165 def test_nrecords_variable(self):
166 "File.nrecords(variable)"
167 for F in [self.file_with_time, self.file_without_time]:
168 for backend in backends:
169 f = PISM.File(ctx.com(), F, backend, PISM.PISM_READONLY,
170 ctx.pio_iosys_id())
171 assert f.nrecords("v", "standard_name", ctx.unit_system()) == 1
172 # found using the standard name
173 assert f.nrecords("w", "standard_name", ctx.unit_system()) == 1
174 assert f.nrecords("missing", "", ctx.unit_system()) == 0
175 f.close()
176
177 def test_nvariables(self):
178 "File.nvariables()"
179 for backend in backends:
180 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
181 ctx.pio_iosys_id())
182 assert f.nvariables() == 4 # time, x, y, v
183 f.close()
184
185 def test_nattributes(self):
186 "File.nattributes()"
187 for backend in backends:
188 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
189 ctx.pio_iosys_id())
190 assert f.nattributes("time") == 4 # units, axis, calendar, long_name
191 assert f.nattributes("x") == 5 # units, axis, long_name, standard_name, spacing_meters
192 assert f.nattributes("PISM_GLOBAL") == 2
193
194 f.close()
195
196 def test_define_dimension(self):
197 "File.define_dimension()"
198 for backend in backends:
199 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE,
200 ctx.pio_iosys_id())
201 f.define_dimension("dim_{}".format(backend), 10 + backend)
202 f.close()
203
204 def test_dimension_length(self):
205 "File.dimension_length()"
206 for backend in backends:
207 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
208 ctx.pio_iosys_id())
209 f.dimension_length("time") == 1
210 f.dimension_length("x") == 3
211 f.dimension_length("y") == 5
212 f.dimension_length("z") == 0
213 f.close()
214
215 def test_dimensions(self):
216 "File.dimensions()"
217 for backend in backends:
218 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE,
219 ctx.pio_iosys_id())
220 assert f.dimensions("v") == ("time", "y", "x")
221
222 variable_name = "scalar_variable_{}".format(backend)
223 f.define_variable(variable_name, PISM.PISM_BYTE, [])
224 assert f.dimensions(variable_name) == ()
225 f.close()
226
227 def test_find_dimension(self):
228 "File.find_dimension()"
229 for backend in backends:
230 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
231 ctx.pio_iosys_id())
232 assert f.find_dimension("x")
233 assert not f.find_dimension("z")
234 f.close()
235
236 def test_dimension_type(self):
237 "File.dimension_type()"
238 for backend in backends:
239 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
240 ctx.pio_iosys_id())
241 f.dimension_type("time", ctx.unit_system()) == PISM.T_AXIS
242 f.dimension_type("x", ctx.unit_system()) == PISM.X_AXIS
243 f.dimension_type("y", ctx.unit_system()) == PISM.Y_AXIS
244
245 try:
246 f.dimension_type("z", ctx.unit_system())
247 fail(backend)
248 except RuntimeError:
249 pass
250
251 f.close()
252
253 f = PISM.File(ctx.com(), self.file_dim_types, backend, PISM.PISM_READONLY,
254 ctx.pio_iosys_id())
255
256 def check(names, axis_type):
257 for c in names:
258 assert f.dimension_type(c, ctx.unit_system()) == axis_type
259
260 check(self.x_names + self.strange_x_names, PISM.X_AXIS)
261 check(self.y_names + self.strange_y_names, PISM.Y_AXIS)
262 check(self.z_names + self.strange_z_names, PISM.Z_AXIS)
263 check(self.t_names + self.strange_t_names, PISM.T_AXIS)
264
265 assert f.dimension_type("unknown_axis", ctx.unit_system()) == PISM.UNKNOWN_AXIS
266
267 f.close()
268
269 def test_read_dimension(self):
270 "File.read_dimension()"
271 for backend in backends:
272 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
273 ctx.pio_iosys_id())
274 assert f.read_dimension("x") == (-10000.0, 0.0, 10000.0)
275
276 try:
277 f.read_dimension("z")
278 fail(backend)
279 except RuntimeError:
280 pass
281
282 f.close()
283
284 def test_variable_name(self):
285 "File.variable_name()"
286 for backend in backends:
287 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
288 ctx.pio_iosys_id())
289 assert f.variable_name(0) == "time"
290
291 # invalid variable index
292 try:
293 f.variable_name(1000)
294 fail(backend)
295 except RuntimeError:
296 pass
297
298 f.close()
299
300 def test_define_variable(self):
301 "File.define_variable()"
302 for backend in backends:
303 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE,
304 ctx.pio_iosys_id())
305 f.define_variable("var_{}".format(backend), PISM.PISM_DOUBLE, ["y", "x"])
306 # defining an existing variable should fail
307 try:
308 f.define_variable("var_{}".format(backend), PISM.PISM_DOUBLE, ["y", "x"])
309 fail(backend)
310 except RuntimeError:
311 pass
312
313 # defining a variable depending on a non-existent dimension should fail gracefully
314 try:
315 f.define_variable("var_{}_1".format(backend), PISM.PISM_DOUBLE, ["y", "x", "z"])
316 fail(backend)
317 except RuntimeError:
318 pass
319 f.close()
320
321 def test_find_variable(self):
322 "File.find_variable(short_name)"
323 for backend in backends:
324 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
325 ctx.pio_iosys_id())
326 assert f.find_variable("time")
327 assert not f.find_variable("z")
328 f.close()
329
330 def test_find_variable(self):
331 "File.find_variable(short_name, standard_name)"
332 for backend in backends:
333 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
334 ctx.pio_iosys_id())
335 assert f.find_variable("v", "standard_name").exists
336 assert f.find_variable("v", "standard_name").found_using_standard_name
337 assert f.find_variable("other_name", "standard_name").found_using_standard_name
338 assert f.find_variable("other_name", "standard_name").name == "v"
339 assert not f.find_variable("v", "").found_using_standard_name
340 assert f.find_variable("missing", "other_standard_name").exists == False
341 assert f.find_variable("missing", "other_standard_name").name == ""
342 f.close()
343
344 f = PISM.File(ctx.com(), self.file_inconsistent, backend, PISM.PISM_READONLY,
345 ctx.pio_iosys_id())
346
347 try:
348 f.find_variable("v", "standard_name")
349 fail(backend)
350 except RuntimeError:
351 pass
352
353 def test_read_variable(self):
354 "File.read_variable()"
355 for backend in backends:
356 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
357 ctx.pio_iosys_id())
358 f.read_variable("x", [1], [1]) == [0.0]
359
360 # start and count of different lengths
361 if PISM.Pism_DEBUG:
362 try:
363 f.read_variable("v", [1, 1], [1, 1, 1])
364 fail(backend)
365 except RuntimeError:
366 pass
367
368 f.close()
369
370 def test_read_variable_transposed(self):
371 raise SkipTest("disable this in Python bindings")
372
373 def test_write_variable(self):
374 "File.write_variable()"
375 for backend in backends:
376 f = PISM.File(ctx.com(), self.file_without_time, backend, PISM.PISM_READWRITE,
377 ctx.pio_iosys_id())
378 f.write_variable("v", [1, 1], [1, 1], [100.0])
379 f.sync()
380 assert f.read_variable("v", [1, 1], [1, 1]) == (100.0,)
381
382 # start and count of different lengths
383 if PISM.Pism_DEBUG:
384 try:
385 f.write_variable("v", [1, 1], [1, 1, 1], [200.0])
386 fail(backend)
387 except RuntimeError:
388 pass
389
390 f.close()
391
392 def test_write_distributed_array(self):
393 raise SkipTest("disable this in Python bindings")
394
395 def test_remove_attribute(self):
396 "File.remove_attribute()"
397 for backend in backends:
398 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE,
399 ctx.pio_iosys_id())
400 assert f.attribute_type("time", "units") == PISM.PISM_CHAR
401 f.remove_attribute("time", "units")
402 assert f.attribute_type("time", "units") == PISM.PISM_NAT
403 f.write_attribute("time", "units", "seconds since 1-1-1")
404 assert f.attribute_type("time", "units") == PISM.PISM_CHAR
405 f.close()
406
407 def test_attribute_name(self):
408 "File.attribute_name()"
409 for backend in backends:
410 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
411 ctx.pio_iosys_id())
412 assert f.attribute_name("time", 0) == "units"
413 assert f.attribute_name("PISM_GLOBAL", 0) == "global_text_attr"
414 f.close()
415
416 def test_attribute_type(self):
417 "File.attribute_type()"
418 for backend in backends:
419 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
420 ctx.pio_iosys_id())
421 f.attribute_type("x", "units") == PISM.PISM_CHAR
422 f.attribute_type("x", "spacing_meters") == PISM.PISM_DOUBLE
423 f.attribute_type("x", "missing") == PISM.PISM_NAT
424 f.attribute_type("PISM_GLOBAL", "global_text_att") == PISM.PISM_CHAR
425 f.close()
426
427 def test_write_attribute_number(self):
428 "File.write_attribute(number)"
429 for backend in backends:
430 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE,
431 ctx.pio_iosys_id())
432 f.write_attribute("v", "new_attribute", PISM.PISM_DOUBLE, (1.0, 2.0))
433 assert f.read_double_attribute("v", "new_attribute") == (1.0, 2.0)
434
435 f.write_attribute("PISM_GLOBAL", "new_attribute", PISM.PISM_DOUBLE, (1.0, 2.0))
436 assert f.read_double_attribute("PISM_GLOBAL", "new_attribute") == (1.0, 2.0)
437
438 f.close()
439
440 def test_write_attribute_string(self):
441 "File.write_attribute(string)"
442 for backend in backends:
443 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE,
444 ctx.pio_iosys_id())
445 f.write_attribute("v", "new_attribute", "test string")
446 assert f.read_text_attribute("v", "new_attribute") == "test string"
447
448 f.write_attribute("PISM_GLOBAL", "new_global_attr", "test_global")
449 assert f.read_text_attribute("PISM_GLOBAL", "new_global_attr") == "test_global"
450 f.close()
451
452 def test_read_double_attribute(self):
453 "File.read_double_attribute()"
454 for backend in backends:
455 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
456 ctx.pio_iosys_id())
457 assert f.read_double_attribute("x", "spacing_meters") == (10000.0,)
458 assert f.read_double_attribute("x", "missing") == ()
459 # type mismatch: fail with a helpful message
460 try:
461 f.read_double_attribute("x", "units")
462 fail(backend)
463 except RuntimeError:
464 pass
465
466 assert f.read_double_attribute("PISM_GLOBAL", "global_double_attr") == (12.0,)
467
468 f.close()
469
470 def test_read_text_attribute(self):
471 "File.read_text_attribute()"
472 for backend in backends:
473 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY,
474 ctx.pio_iosys_id())
475 assert f.read_text_attribute("x", "units") == "m"
476 assert f.read_text_attribute("x", "missing") == ""
477
478 assert f.read_text_attribute("PISM_GLOBAL", "global_text_attr") == "test_global"
479
480 # throw an error in case of a type mismatch
481 try:
482 f.read_text_attribute("x", "spacing_meters")
483 fail(backend)
484 except RuntimeError:
485 pass
486
487 f.close()
488
489 def test_append_history(self):
490 "File.read_text_attribute()"
491 for backend in backends:
492 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE,
493 ctx.pio_iosys_id())
494 try:
495 f.remove_attribute("PISM_GLOBAL", "history")
496 except:
497 pass
498 f.append_history("one")
499 f.append_history("two")
500 assert f.read_text_attribute("PISM_GLOBAL", "history") == "twoone"
501 f.close()
502
503 def setUp(self):
504 self.file_with_time = "test_file_with_time.nc"
505 self.file_without_time = "test_file_without_time.nc"
506 self.file_inconsistent = "test_file_inconsistent.nc"
507 self.file_dim_types = "test_file_dim_types.nc"
508
509 self.files = [self.file_with_time, self.file_without_time, self.file_inconsistent]
510
511 grid = PISM.testing.shallow_grid()
512 vec = PISM.IceModelVec2S(grid, "v", PISM.WITHOUT_GHOSTS)
513 vec.set_attrs("testing", "dummy variable for testing",
514 "Kelvin", "Celsius", "standard_name", 0)
515 vec.set(1.0)
516 vec.set_time_independent(True)
517 vec.dump(self.file_without_time)
518
519 # file with two variables with the same standard name
520 vec.dump(self.file_inconsistent)
521 vec.metadata(0).set_name("w")
522 vec.write(self.file_inconsistent)
523 vec.metadata(0).set_name("v")
524
525 vec.set(2.0)
526 vec.set_time_independent(False)
527 vec.dump(self.file_with_time)
528
529 f = PISM.File(ctx.com(), self.file_with_time, PISM.PISM_NETCDF3, PISM.PISM_READWRITE)
530 f.write_attribute("PISM_GLOBAL", "global_text_attr", "test_global")
531 f.write_attribute("PISM_GLOBAL", "global_double_attr", PISM.PISM_DOUBLE, [12.0])
532 f.close()
533
534 f = PISM.File(ctx.com(), self.file_dim_types, PISM.PISM_NETCDF3, PISM.PISM_READWRITE_CLOBBER)
535 self.x_names = ["x", "X", "x1", "X1"]
536 strange_x_attrs = {"coord_x_1" : ("axis", "x"),
537 "coord_x_2" : ("axis", "X")}
538 self.strange_x_names = list(strange_x_attrs.keys())
539
540 self.y_names = ["y", "Y", "y1", "Y1"]
541 strange_y_attrs = {"coord_y_1" : ("axis", "y"),
542 "coord_y_2" : ("axis", "Y")}
543 self.strange_y_names = list(strange_y_attrs.keys())
544
545 self.z_names = ["z", "Z", "z1", "Z1"]
546 strange_z_attrs = {"coord_z_1" : ("axis", "z"),
547 "coord_z_2" : ("axis", "Z")}
548 self.strange_z_names = list(strange_z_attrs.keys())
549
550 self.t_names = ["t", "T", "time", "t0", "T0"]
551 strange_t_attrs = {"coord_t_1" : ("units", "years"),
552 "coord_t_2" : ("standard_name", "time"),
553 "coord_t_3" : ("axis", "T"),
554 "coord_t_4" : ("axis", "t")}
555 self.strange_t_names = list(strange_t_attrs.keys())
556
557 def create(names, strange_names, attrs):
558 for v in names + strange_names:
559 f.define_variable(v, PISM.PISM_DOUBLE, [])
560
561 for c, (a, v) in attrs.items():
562 f.write_attribute(c, a, v)
563
564 create(self.x_names, self.strange_x_names, strange_x_attrs)
565 create(self.y_names, self.strange_y_names, strange_y_attrs)
566 create(self.z_names, self.strange_z_names, strange_z_attrs)
567 create(self.t_names, self.strange_t_names, strange_t_attrs)
568
569 f.define_variable("unknown_axis", PISM.PISM_DOUBLE, [])
570
571 def tearDown(self):
572 for f in self.files:
573 os.remove(f)
574 pass
575
576 class StringAttribute(TestCase):
577 "Test reading a NetCDF-4 string attribute."
578
579 def test_read_string_attribute(self):
580 "File.read_text_attribute() (string)"
581
582 backends = [PISM.PISM_NETCDF3]
583
584 if PISM.Pism_USE_PARALLEL_NETCDF4:
585 backends += [PISM.PISM_NETCDF4_PARALLEL]
586
587 for backend in backends:
588 f = PISM.File(ctx.com(), self.basename + ".nc", backend, PISM.PISM_READONLY)
589 assert self.attribute == f.read_text_attribute("PISM_GLOBAL", "string_attribute")
590 assert self.attribute == f.read_text_attribute("PISM_GLOBAL", "text_attribute")
591 # multi-valued string attributes are turned into comma-separated lists
592 assert "{0},{0}".format(self.attribute) == f.read_text_attribute("PISM_GLOBAL",
593 "string_multi_value")
594 f.close()
595
596 def setUp(self):
597 self.basename = "string_attribute_test"
598 self.attribute = "string attribute"
599
600 cdl = """
601 netcdf string_attribute_test {{
602 string :string_attribute = "{0}" ;
603 string :string_multi_value = "{0}", "{0}" ;
604 :text_attribute = "{0}" ;
605 }}
606 """.format(self.attribute)
607 with open(self.basename + ".cdl", "w") as f:
608 f.write(cdl)
609
610 os.system("ncgen -4 %s.cdl" % self.basename)
611
612 def tearDown(self):
613 os.remove(self.basename + ".nc")
614 os.remove(self.basename + ".cdl")