tfile.py - pism - [fork] customized build of PISM, the parallel ice sheet model (tillflux branch) (HTM) git clone git://src.adamsgaard.dk/pism (DIR) Log (DIR) Files (DIR) Refs (DIR) LICENSE --- tfile.py (23540B) --- 1 import PISM 2 import PISM.testing 3 import os 4 from unittest import TestCase, SkipTest 5 6 # always available 7 backends = [PISM.PISM_NETCDF3] 8 9 if PISM.Pism_USE_PARALLEL_NETCDF4: 10 backends += [PISM.PISM_NETCDF4_PARALLEL] 11 12 if PISM.Pism_USE_PNETCDF: 13 backends += [PISM.PISM_PNETCDF] 14 15 if PISM.Pism_USE_PIO: 16 # assume that ParallelIO was built with all possible libraries 17 backends += [PISM.PISM_PIO_PNETCDF, PISM.PISM_PIO_NETCDF, 18 PISM.PISM_PIO_NETCDF4C, PISM.PISM_PIO_NETCDF4P] 19 20 ctx = PISM.Context().ctx 21 22 backend_names = {PISM.PISM_NETCDF3 : "netcdf3", 23 PISM.PISM_NETCDF4_PARALLEL : "netcdf4_parallel", 24 PISM.PISM_PNETCDF : "pnetcdf", 25 PISM.PISM_PIO_NETCDF : "pio_netcdf", 26 PISM.PISM_PIO_NETCDF4P : "pio_netcdf4p", 27 PISM.PISM_PIO_NETCDF4C : "pio_netcdf4c", 28 PISM.PISM_PIO_PNETCDF: "pio_pnetcdf"} 29 30 def fail(backend): 31 assert False, "test failed (backend = {})".format(backend_names[backend]) 32 33 def test_string_to_backend(): 34 "PISM.string_to_backend()" 35 36 for backend, name in backend_names.items(): 37 assert PISM.string_to_backend(name) == backend 38 39 try: 40 PISM.string_to_backend("invalid") 41 return False 42 except RuntimeError: 43 pass 44 45 class File(TestCase): 46 47 def test_empty_filename(self): 48 for backend in backends: 49 try: 50 f = PISM.File(ctx.com(), "", backend, PISM.PISM_READONLY, 51 ctx.pio_iosys_id()) 52 fail(backend) 53 except RuntimeError: 54 pass 55 56 def test_missing_file(self): 57 for backend in backends: 58 try: 59 f = PISM.File(ctx.com(), "missing_file.nc", backend, PISM.PISM_READONLY, 60 ctx.pio_iosys_id()) 61 fail(backend) 62 except RuntimeError: 63 pass 64 65 def test_backend_guessing(self): 66 "File(..., PISM_GUESS, ...)" 67 68 f = PISM.File(ctx.com(), self.file_with_time, PISM.PISM_GUESS, PISM.PISM_READONLY, 69 ctx.pio_iosys_id()) 70 assert f.nrecords() == 1 71 72 def test_create_clobber(self): 73 "File(..., PISM_READWRITE_CLOBBER)" 74 try: 75 for backend in backends: 76 f = PISM.File(ctx.com(), "test_filename.nc", backend, PISM.PISM_READWRITE_CLOBBER, 77 ctx.pio_iosys_id()) 78 finally: 79 os.remove("test_filename.nc") 80 81 def test_create_move(self): 82 "File(..., PISM_READWRITE_MOVE)" 83 try: 84 for backend in backends: 85 f = PISM.File(ctx.com(), "test_filename.nc", backend, PISM.PISM_READWRITE_MOVE, 86 ctx.pio_iosys_id()) 87 finally: 88 os.remove("test_filename.nc") 89 try: 90 os.remove("test_filename.nc~") 91 except: 92 pass 93 94 def test_backend(self): 95 "File.backend()" 96 for backend in backends: 97 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 98 ctx.pio_iosys_id()) 99 assert f.backend() == backend 100 f.close() 101 102 def test_com(self): 103 "File.com()" 104 for backend in backends: 105 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 106 ctx.pio_iosys_id()) 107 assert f.com() == ctx.com() 108 f.close() 109 110 def test_close(self): 111 "File.close()" 112 for backend in backends: 113 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 114 ctx.pio_iosys_id()) 115 f.close() 116 117 try: 118 f.close() 119 # closing twice is an error 120 fail(backend) 121 except RuntimeError: 122 pass 123 124 def test_redef(self): 125 "File.redef()" 126 for backend in backends: 127 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE, 128 ctx.pio_iosys_id()) 129 f.redef() 130 f.close() 131 132 def test_enddef(self): 133 "File.enddef()" 134 for backend in backends: 135 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE, 136 ctx.pio_iosys_id()) 137 f.enddef() 138 f.close() 139 140 def test_sync(self): 141 "File.sync()" 142 for backend in backends: 143 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE, 144 ctx.pio_iosys_id()) 145 f.sync() 146 f.close() 147 148 def test_filename(self): 149 "File.filename()" 150 for backend in backends: 151 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 152 ctx.pio_iosys_id()) 153 assert f.filename() == self.file_with_time 154 f.close() 155 156 def test_nrecords(self): 157 "File.nrecords()" 158 for F in self.files: 159 for backend in backends: 160 f = PISM.File(ctx.com(), F, backend, PISM.PISM_READONLY, 161 ctx.pio_iosys_id()) 162 assert f.nrecords() == 1 163 f.close() 164 165 def test_nrecords_variable(self): 166 "File.nrecords(variable)" 167 for F in [self.file_with_time, self.file_without_time]: 168 for backend in backends: 169 f = PISM.File(ctx.com(), F, backend, PISM.PISM_READONLY, 170 ctx.pio_iosys_id()) 171 assert f.nrecords("v", "standard_name", ctx.unit_system()) == 1 172 # found using the standard name 173 assert f.nrecords("w", "standard_name", ctx.unit_system()) == 1 174 assert f.nrecords("missing", "", ctx.unit_system()) == 0 175 f.close() 176 177 def test_nvariables(self): 178 "File.nvariables()" 179 for backend in backends: 180 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 181 ctx.pio_iosys_id()) 182 assert f.nvariables() == 4 # time, x, y, v 183 f.close() 184 185 def test_nattributes(self): 186 "File.nattributes()" 187 for backend in backends: 188 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 189 ctx.pio_iosys_id()) 190 assert f.nattributes("time") == 4 # units, axis, calendar, long_name 191 assert f.nattributes("x") == 5 # units, axis, long_name, standard_name, spacing_meters 192 assert f.nattributes("PISM_GLOBAL") == 2 193 194 f.close() 195 196 def test_define_dimension(self): 197 "File.define_dimension()" 198 for backend in backends: 199 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE, 200 ctx.pio_iosys_id()) 201 f.define_dimension("dim_{}".format(backend), 10 + backend) 202 f.close() 203 204 def test_dimension_length(self): 205 "File.dimension_length()" 206 for backend in backends: 207 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 208 ctx.pio_iosys_id()) 209 f.dimension_length("time") == 1 210 f.dimension_length("x") == 3 211 f.dimension_length("y") == 5 212 f.dimension_length("z") == 0 213 f.close() 214 215 def test_dimensions(self): 216 "File.dimensions()" 217 for backend in backends: 218 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE, 219 ctx.pio_iosys_id()) 220 assert f.dimensions("v") == ("time", "y", "x") 221 222 variable_name = "scalar_variable_{}".format(backend) 223 f.define_variable(variable_name, PISM.PISM_BYTE, []) 224 assert f.dimensions(variable_name) == () 225 f.close() 226 227 def test_find_dimension(self): 228 "File.find_dimension()" 229 for backend in backends: 230 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 231 ctx.pio_iosys_id()) 232 assert f.find_dimension("x") 233 assert not f.find_dimension("z") 234 f.close() 235 236 def test_dimension_type(self): 237 "File.dimension_type()" 238 for backend in backends: 239 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 240 ctx.pio_iosys_id()) 241 f.dimension_type("time", ctx.unit_system()) == PISM.T_AXIS 242 f.dimension_type("x", ctx.unit_system()) == PISM.X_AXIS 243 f.dimension_type("y", ctx.unit_system()) == PISM.Y_AXIS 244 245 try: 246 f.dimension_type("z", ctx.unit_system()) 247 fail(backend) 248 except RuntimeError: 249 pass 250 251 f.close() 252 253 f = PISM.File(ctx.com(), self.file_dim_types, backend, PISM.PISM_READONLY, 254 ctx.pio_iosys_id()) 255 256 def check(names, axis_type): 257 for c in names: 258 assert f.dimension_type(c, ctx.unit_system()) == axis_type 259 260 check(self.x_names + self.strange_x_names, PISM.X_AXIS) 261 check(self.y_names + self.strange_y_names, PISM.Y_AXIS) 262 check(self.z_names + self.strange_z_names, PISM.Z_AXIS) 263 check(self.t_names + self.strange_t_names, PISM.T_AXIS) 264 265 assert f.dimension_type("unknown_axis", ctx.unit_system()) == PISM.UNKNOWN_AXIS 266 267 f.close() 268 269 def test_read_dimension(self): 270 "File.read_dimension()" 271 for backend in backends: 272 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 273 ctx.pio_iosys_id()) 274 assert f.read_dimension("x") == (-10000.0, 0.0, 10000.0) 275 276 try: 277 f.read_dimension("z") 278 fail(backend) 279 except RuntimeError: 280 pass 281 282 f.close() 283 284 def test_variable_name(self): 285 "File.variable_name()" 286 for backend in backends: 287 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 288 ctx.pio_iosys_id()) 289 assert f.variable_name(0) == "time" 290 291 # invalid variable index 292 try: 293 f.variable_name(1000) 294 fail(backend) 295 except RuntimeError: 296 pass 297 298 f.close() 299 300 def test_define_variable(self): 301 "File.define_variable()" 302 for backend in backends: 303 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE, 304 ctx.pio_iosys_id()) 305 f.define_variable("var_{}".format(backend), PISM.PISM_DOUBLE, ["y", "x"]) 306 # defining an existing variable should fail 307 try: 308 f.define_variable("var_{}".format(backend), PISM.PISM_DOUBLE, ["y", "x"]) 309 fail(backend) 310 except RuntimeError: 311 pass 312 313 # defining a variable depending on a non-existent dimension should fail gracefully 314 try: 315 f.define_variable("var_{}_1".format(backend), PISM.PISM_DOUBLE, ["y", "x", "z"]) 316 fail(backend) 317 except RuntimeError: 318 pass 319 f.close() 320 321 def test_find_variable(self): 322 "File.find_variable(short_name)" 323 for backend in backends: 324 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 325 ctx.pio_iosys_id()) 326 assert f.find_variable("time") 327 assert not f.find_variable("z") 328 f.close() 329 330 def test_find_variable(self): 331 "File.find_variable(short_name, standard_name)" 332 for backend in backends: 333 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 334 ctx.pio_iosys_id()) 335 assert f.find_variable("v", "standard_name").exists 336 assert f.find_variable("v", "standard_name").found_using_standard_name 337 assert f.find_variable("other_name", "standard_name").found_using_standard_name 338 assert f.find_variable("other_name", "standard_name").name == "v" 339 assert not f.find_variable("v", "").found_using_standard_name 340 assert f.find_variable("missing", "other_standard_name").exists == False 341 assert f.find_variable("missing", "other_standard_name").name == "" 342 f.close() 343 344 f = PISM.File(ctx.com(), self.file_inconsistent, backend, PISM.PISM_READONLY, 345 ctx.pio_iosys_id()) 346 347 try: 348 f.find_variable("v", "standard_name") 349 fail(backend) 350 except RuntimeError: 351 pass 352 353 def test_read_variable(self): 354 "File.read_variable()" 355 for backend in backends: 356 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 357 ctx.pio_iosys_id()) 358 f.read_variable("x", [1], [1]) == [0.0] 359 360 # start and count of different lengths 361 if PISM.Pism_DEBUG: 362 try: 363 f.read_variable("v", [1, 1], [1, 1, 1]) 364 fail(backend) 365 except RuntimeError: 366 pass 367 368 f.close() 369 370 def test_read_variable_transposed(self): 371 raise SkipTest("disable this in Python bindings") 372 373 def test_write_variable(self): 374 "File.write_variable()" 375 for backend in backends: 376 f = PISM.File(ctx.com(), self.file_without_time, backend, PISM.PISM_READWRITE, 377 ctx.pio_iosys_id()) 378 f.write_variable("v", [1, 1], [1, 1], [100.0]) 379 f.sync() 380 assert f.read_variable("v", [1, 1], [1, 1]) == (100.0,) 381 382 # start and count of different lengths 383 if PISM.Pism_DEBUG: 384 try: 385 f.write_variable("v", [1, 1], [1, 1, 1], [200.0]) 386 fail(backend) 387 except RuntimeError: 388 pass 389 390 f.close() 391 392 def test_write_distributed_array(self): 393 raise SkipTest("disable this in Python bindings") 394 395 def test_remove_attribute(self): 396 "File.remove_attribute()" 397 for backend in backends: 398 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE, 399 ctx.pio_iosys_id()) 400 assert f.attribute_type("time", "units") == PISM.PISM_CHAR 401 f.remove_attribute("time", "units") 402 assert f.attribute_type("time", "units") == PISM.PISM_NAT 403 f.write_attribute("time", "units", "seconds since 1-1-1") 404 assert f.attribute_type("time", "units") == PISM.PISM_CHAR 405 f.close() 406 407 def test_attribute_name(self): 408 "File.attribute_name()" 409 for backend in backends: 410 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 411 ctx.pio_iosys_id()) 412 assert f.attribute_name("time", 0) == "units" 413 assert f.attribute_name("PISM_GLOBAL", 0) == "global_text_attr" 414 f.close() 415 416 def test_attribute_type(self): 417 "File.attribute_type()" 418 for backend in backends: 419 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 420 ctx.pio_iosys_id()) 421 f.attribute_type("x", "units") == PISM.PISM_CHAR 422 f.attribute_type("x", "spacing_meters") == PISM.PISM_DOUBLE 423 f.attribute_type("x", "missing") == PISM.PISM_NAT 424 f.attribute_type("PISM_GLOBAL", "global_text_att") == PISM.PISM_CHAR 425 f.close() 426 427 def test_write_attribute_number(self): 428 "File.write_attribute(number)" 429 for backend in backends: 430 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE, 431 ctx.pio_iosys_id()) 432 f.write_attribute("v", "new_attribute", PISM.PISM_DOUBLE, (1.0, 2.0)) 433 assert f.read_double_attribute("v", "new_attribute") == (1.0, 2.0) 434 435 f.write_attribute("PISM_GLOBAL", "new_attribute", PISM.PISM_DOUBLE, (1.0, 2.0)) 436 assert f.read_double_attribute("PISM_GLOBAL", "new_attribute") == (1.0, 2.0) 437 438 f.close() 439 440 def test_write_attribute_string(self): 441 "File.write_attribute(string)" 442 for backend in backends: 443 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE, 444 ctx.pio_iosys_id()) 445 f.write_attribute("v", "new_attribute", "test string") 446 assert f.read_text_attribute("v", "new_attribute") == "test string" 447 448 f.write_attribute("PISM_GLOBAL", "new_global_attr", "test_global") 449 assert f.read_text_attribute("PISM_GLOBAL", "new_global_attr") == "test_global" 450 f.close() 451 452 def test_read_double_attribute(self): 453 "File.read_double_attribute()" 454 for backend in backends: 455 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 456 ctx.pio_iosys_id()) 457 assert f.read_double_attribute("x", "spacing_meters") == (10000.0,) 458 assert f.read_double_attribute("x", "missing") == () 459 # type mismatch: fail with a helpful message 460 try: 461 f.read_double_attribute("x", "units") 462 fail(backend) 463 except RuntimeError: 464 pass 465 466 assert f.read_double_attribute("PISM_GLOBAL", "global_double_attr") == (12.0,) 467 468 f.close() 469 470 def test_read_text_attribute(self): 471 "File.read_text_attribute()" 472 for backend in backends: 473 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READONLY, 474 ctx.pio_iosys_id()) 475 assert f.read_text_attribute("x", "units") == "m" 476 assert f.read_text_attribute("x", "missing") == "" 477 478 assert f.read_text_attribute("PISM_GLOBAL", "global_text_attr") == "test_global" 479 480 # throw an error in case of a type mismatch 481 try: 482 f.read_text_attribute("x", "spacing_meters") 483 fail(backend) 484 except RuntimeError: 485 pass 486 487 f.close() 488 489 def test_append_history(self): 490 "File.read_text_attribute()" 491 for backend in backends: 492 f = PISM.File(ctx.com(), self.file_with_time, backend, PISM.PISM_READWRITE, 493 ctx.pio_iosys_id()) 494 try: 495 f.remove_attribute("PISM_GLOBAL", "history") 496 except: 497 pass 498 f.append_history("one") 499 f.append_history("two") 500 assert f.read_text_attribute("PISM_GLOBAL", "history") == "twoone" 501 f.close() 502 503 def setUp(self): 504 self.file_with_time = "test_file_with_time.nc" 505 self.file_without_time = "test_file_without_time.nc" 506 self.file_inconsistent = "test_file_inconsistent.nc" 507 self.file_dim_types = "test_file_dim_types.nc" 508 509 self.files = [self.file_with_time, self.file_without_time, self.file_inconsistent] 510 511 grid = PISM.testing.shallow_grid() 512 vec = PISM.IceModelVec2S(grid, "v", PISM.WITHOUT_GHOSTS) 513 vec.set_attrs("testing", "dummy variable for testing", 514 "Kelvin", "Celsius", "standard_name", 0) 515 vec.set(1.0) 516 vec.set_time_independent(True) 517 vec.dump(self.file_without_time) 518 519 # file with two variables with the same standard name 520 vec.dump(self.file_inconsistent) 521 vec.metadata(0).set_name("w") 522 vec.write(self.file_inconsistent) 523 vec.metadata(0).set_name("v") 524 525 vec.set(2.0) 526 vec.set_time_independent(False) 527 vec.dump(self.file_with_time) 528 529 f = PISM.File(ctx.com(), self.file_with_time, PISM.PISM_NETCDF3, PISM.PISM_READWRITE) 530 f.write_attribute("PISM_GLOBAL", "global_text_attr", "test_global") 531 f.write_attribute("PISM_GLOBAL", "global_double_attr", PISM.PISM_DOUBLE, [12.0]) 532 f.close() 533 534 f = PISM.File(ctx.com(), self.file_dim_types, PISM.PISM_NETCDF3, PISM.PISM_READWRITE_CLOBBER) 535 self.x_names = ["x", "X", "x1", "X1"] 536 strange_x_attrs = {"coord_x_1" : ("axis", "x"), 537 "coord_x_2" : ("axis", "X")} 538 self.strange_x_names = list(strange_x_attrs.keys()) 539 540 self.y_names = ["y", "Y", "y1", "Y1"] 541 strange_y_attrs = {"coord_y_1" : ("axis", "y"), 542 "coord_y_2" : ("axis", "Y")} 543 self.strange_y_names = list(strange_y_attrs.keys()) 544 545 self.z_names = ["z", "Z", "z1", "Z1"] 546 strange_z_attrs = {"coord_z_1" : ("axis", "z"), 547 "coord_z_2" : ("axis", "Z")} 548 self.strange_z_names = list(strange_z_attrs.keys()) 549 550 self.t_names = ["t", "T", "time", "t0", "T0"] 551 strange_t_attrs = {"coord_t_1" : ("units", "years"), 552 "coord_t_2" : ("standard_name", "time"), 553 "coord_t_3" : ("axis", "T"), 554 "coord_t_4" : ("axis", "t")} 555 self.strange_t_names = list(strange_t_attrs.keys()) 556 557 def create(names, strange_names, attrs): 558 for v in names + strange_names: 559 f.define_variable(v, PISM.PISM_DOUBLE, []) 560 561 for c, (a, v) in attrs.items(): 562 f.write_attribute(c, a, v) 563 564 create(self.x_names, self.strange_x_names, strange_x_attrs) 565 create(self.y_names, self.strange_y_names, strange_y_attrs) 566 create(self.z_names, self.strange_z_names, strange_z_attrs) 567 create(self.t_names, self.strange_t_names, strange_t_attrs) 568 569 f.define_variable("unknown_axis", PISM.PISM_DOUBLE, []) 570 571 def tearDown(self): 572 for f in self.files: 573 os.remove(f) 574 pass 575 576 class StringAttribute(TestCase): 577 "Test reading a NetCDF-4 string attribute." 578 579 def test_read_string_attribute(self): 580 "File.read_text_attribute() (string)" 581 582 backends = [PISM.PISM_NETCDF3] 583 584 if PISM.Pism_USE_PARALLEL_NETCDF4: 585 backends += [PISM.PISM_NETCDF4_PARALLEL] 586 587 for backend in backends: 588 f = PISM.File(ctx.com(), self.basename + ".nc", backend, PISM.PISM_READONLY) 589 assert self.attribute == f.read_text_attribute("PISM_GLOBAL", "string_attribute") 590 assert self.attribute == f.read_text_attribute("PISM_GLOBAL", "text_attribute") 591 # multi-valued string attributes are turned into comma-separated lists 592 assert "{0},{0}".format(self.attribute) == f.read_text_attribute("PISM_GLOBAL", 593 "string_multi_value") 594 f.close() 595 596 def setUp(self): 597 self.basename = "string_attribute_test" 598 self.attribute = "string attribute" 599 600 cdl = """ 601 netcdf string_attribute_test {{ 602 string :string_attribute = "{0}" ; 603 string :string_multi_value = "{0}", "{0}" ; 604 :text_attribute = "{0}" ; 605 }} 606 """.format(self.attribute) 607 with open(self.basename + ".cdl", "w") as f: 608 f.write(cdl) 609 610 os.system("ncgen -4 %s.cdl" % self.basename) 611 612 def tearDown(self): 613 os.remove(self.basename + ".nc") 614 os.remove(self.basename + ".cdl")