001/*- 002 ******************************************************************************* 003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd. 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the Eclipse Public License v1.0 006 * which accompanies this distribution, and is available at 007 * http://www.eclipse.org/legal/epl-v10.html 008 * 009 * Contributors: 010 * Peter Chang - initial API and implementation and/or initial documentation 011 *******************************************************************************/ 012 013package org.eclipse.january.dataset; 014 015import java.io.Serializable; 016import java.lang.annotation.Annotation; 017import java.lang.reflect.Array; 018import java.lang.reflect.Field; 019import java.util.ArrayList; 020import java.util.Arrays; 021import java.util.Collection; 022import java.util.concurrent.ConcurrentMap; 023import java.util.concurrent.ConcurrentHashMap; 024import java.util.List; 025import java.util.Map; 026 027import org.eclipse.january.DatasetException; 028import org.eclipse.january.MetadataException; 029import org.eclipse.january.metadata.Dirtiable; 030import org.eclipse.january.metadata.ErrorMetadata; 031import org.eclipse.january.metadata.IMetadata; 032import org.eclipse.january.metadata.MetadataFactory; 033import org.eclipse.january.metadata.MetadataType; 034import org.eclipse.january.metadata.Reshapeable; 035import org.eclipse.january.metadata.Sliceable; 036import org.eclipse.january.metadata.Transposable; 037import org.slf4j.Logger; 038import org.slf4j.LoggerFactory; 039 040/** 041 * Common base for both lazy and normal dataset implementations 042 */ 043public abstract class LazyDatasetBase implements ILazyDataset, Serializable { 044 045 private static final long serialVersionUID = 767926846438976050L; 046 047 protected static final Logger logger = LoggerFactory.getLogger(LazyDatasetBase.class); 048 049 protected static boolean catchExceptions; 050 051 static { 052 /** 053 * Boolean to set to true if running jython scripts that utilise ScisoftPy in IDE 054 */ 055 catchExceptions = Boolean.getBoolean("run.in.eclipse"); 056 } 057 058 transient private boolean dirty = true; // indicate dirty state of metadata 059 protected String name = ""; 060 061 /** 062 * The shape or dimensions of the dataset 063 */ 064 protected int[] shape; 065 066 protected ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> metadata = null; 067 068 /** 069 * @return type of dataset item 070 */ 071 abstract public int getDType(); 072 073 @Override 074 public Class<?> getElementClass() { 075 return DTypeUtils.getElementClass(getDType()); 076 } 077 078 @Override 079 public LazyDatasetBase clone() { 080 return null; 081 } 082 083 @Override 084 public boolean equals(Object obj) { 085 if (this == obj) { 086 return true; 087 } 088 if (obj == null) { 089 return false; 090 } 091 if (!getClass().equals(obj.getClass())) { 092 return false; 093 } 094 095 LazyDatasetBase other = (LazyDatasetBase) obj; 096 if (getDType() != other.getDType()) { 097 return false; 098 } 099 if (getElementsPerItem() != other.getElementsPerItem()) { 100 return false; 101 } 102 if (!Arrays.equals(shape, other.shape)) { 103 return false; 104 } 105 return true; 106 } 107 108 @Override 109 public int hashCode() { 110 int hash = getDType() * 17 + getElementsPerItem(); 111 int rank = shape.length; 112 for (int i = 0; i < rank; i++) { 113 hash = hash*17 + shape[i]; 114 } 115 return hash; 116 } 117 118 @Override 119 public String getName() { 120 return name; 121 } 122 123 @Override 124 public void setName(String name) { 125 this.name = name; 126 } 127 128 @Override 129 public int[] getShape() { 130 return shape.clone(); 131 } 132 133 @Override 134 public int getRank() { 135 return shape.length; 136 } 137 138 /** 139 * This method allows anything that dirties the dataset to clear various metadata values 140 * so that the other methods can work correctly. 141 */ 142 public void setDirty() { 143 dirty = true; 144 } 145 146 /** 147 * Find first sub-interface of (or class that directly implements) MetadataType 148 * @param clazz 149 * @return sub-interface 150 * @exception IllegalArgumentException when given class is {@link MetadataType} or an anonymous sub-class of it 151 */ 152 @SuppressWarnings("unchecked") 153 public static Class<? extends MetadataType> findMetadataTypeSubInterfaces(Class<? extends MetadataType> clazz) { 154 if (clazz.equals(MetadataType.class)) { 155 throw new IllegalArgumentException("Cannot accept MetadataType"); 156 } 157 158 if (clazz.isInterface()) { 159 return clazz; 160 } 161 162 if (clazz.isAnonymousClass()) { // special case 163 Class<?> s = clazz.getSuperclass(); 164 if (!s.equals(Object.class)) { 165 // only use super class if it is not an anonymous class of an interface 166 clazz = (Class<? extends MetadataType>) s; 167 } 168 } 169 170 for (Class<?> c : clazz.getInterfaces()) { 171 if (c.equals(MetadataType.class)) { 172 if (clazz.isAnonymousClass()) { 173 throw new IllegalArgumentException("Cannot accept anonymous subclasses of MetadataType"); 174 } 175 return clazz; 176 } 177 if (MetadataType.class.isAssignableFrom(c)) { 178 return (Class<? extends MetadataType>) c; 179 } 180 } 181 182 Class<?> c = clazz.getSuperclass(); // Naughty: someone has sub-classed a metadata class 183 if (c != null) { 184 return findMetadataTypeSubInterfaces((Class<? extends MetadataType>) c); 185 } 186 187 logger.error("Somehow the search for metadata type interface ended in a bad place"); 188 assert false; // should not be able to get here!!! 189 return null; 190 } 191 192 @Override 193 public void setMetadata(MetadataType metadata) { 194 addMetadata(metadata, true); 195 } 196 197 @Override 198 public void addMetadata(MetadataType metadata) { 199 addMetadata(metadata, false); 200 } 201 202 private synchronized void addMetadata(MetadataType metadata, boolean clear) { 203 if (metadata == null) 204 return; 205 206 if (this.metadata == null) { 207 this.metadata = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 208 } 209 210 Class<? extends MetadataType> clazz = findMetadataTypeSubInterfaces(metadata.getClass()); 211 if (!this.metadata.containsKey(clazz)) { 212 this.metadata.put(clazz, new ArrayList<MetadataType>()); 213 } else if (clear) { 214 this.metadata.get(clazz).clear(); 215 } 216 this.metadata.get(clazz).add(metadata); 217 218 // add for special case of sub-interfaces of IMetadata 219 if (!IMetadata.class.equals(clazz) && IMetadata.class.isAssignableFrom(clazz)) { 220 clazz = IMetadata.class; 221 if (!this.metadata.containsKey(clazz)) { 222 this.metadata.put(clazz, new ArrayList<MetadataType>()); 223 } else if (clear) { 224 this.metadata.get(clazz).clear(); 225 } 226 this.metadata.get(clazz).add(metadata); 227 } 228 } 229 230 @Override 231 @Deprecated 232 public synchronized IMetadata getMetadata() { 233 return getFirstMetadata(IMetadata.class); 234 } 235 236 @SuppressWarnings("unchecked") 237 @Override 238 public synchronized <S extends MetadataType, T extends S> List<S> getMetadata(Class<T> clazz) throws MetadataException { 239 if (metadata == null) 240 return null; 241 242 if (dirty) { 243 dirtyMetadata(); 244 dirty = false; 245 } 246 247 if (clazz == null) { 248 List<S> all = new ArrayList<S>(); 249 for (Class<? extends MetadataType> c : metadata.keySet()) { 250 all.addAll((Collection<S>) metadata.get(c)); 251 } 252 return all; 253 } 254 255 return (List<S>) metadata.get(findMetadataTypeSubInterfaces(clazz)); 256 } 257 258 @Override 259 public synchronized <S extends MetadataType, T extends S> S getFirstMetadata(Class<T> clazz) { 260 try { 261 List<S> ml = getMetadata(clazz); 262 if (ml == null) return null; 263 for (S t : ml) { 264 if (clazz.isInstance(t)) return t; 265 } 266 } catch (Exception e) { 267 logger.error("Get metadata failed!",e); 268 } 269 270 return null; 271 } 272 273 @Override 274 public synchronized void clearMetadata(Class<? extends MetadataType> clazz) { 275 if (metadata == null) 276 return; 277 278 if (clazz == null) { 279 metadata.clear(); 280 return; 281 } 282 283 List<MetadataType> list = metadata.get(findMetadataTypeSubInterfaces(clazz)); 284 if( list != null) { 285 list.clear(); 286 } 287 } 288 289 /** 290 * @since 2.0 291 */ 292 protected synchronized ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata() { 293 return copyMetadata(metadata); 294 } 295 296 /** 297 * @since 2.0 298 */ 299 protected static ConcurrentMap<Class<? extends MetadataType>, List<MetadataType>> copyMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> metadata) { 300 if (metadata == null) 301 return null; 302 303 ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>> map = new ConcurrentHashMap<Class<? extends MetadataType>, List<MetadataType>>(); 304 305 for (Class<? extends MetadataType> c : metadata.keySet()) { 306 List<MetadataType> l = metadata.get(c); 307 List<MetadataType> nl = new ArrayList<MetadataType>(l.size()); 308 map.put(c, nl); 309 for (MetadataType m : l) { 310 nl.add(m.clone()); 311 } 312 } 313 return map; 314 } 315 316 interface MetadatasetAnnotationOperation { 317 /** 318 * Process value of given field 319 * <p> 320 * When the field is not a container then the returned value 321 * may replace the old value 322 * @param f given field 323 * @param o value of field 324 * @return transformed field 325 */ 326 Object processField(Field f, Object o); 327 328 /** 329 * @return annotated class 330 */ 331 Class<? extends Annotation> getAnnClass(); 332 333 /** 334 * @param axis 335 * @return number of dimensions to insert or remove 336 */ 337 int change(int axis); 338 339 /** 340 * 341 * @return rank or -1 to match 342 */ 343 int getNewRank(); 344 345 /** 346 * Run on given lazy dataset 347 * @param lz 348 * @return 349 */ 350 ILazyDataset run(ILazyDataset lz); 351 } 352 353 class MdsSlice implements MetadatasetAnnotationOperation { 354 private boolean asView; 355 private int[] start; 356 private int[] stop; 357 private int[] step; 358 private int[] oShape; 359 private long oSize; 360 361 public MdsSlice(boolean asView, final int[] start, final int[] stop, final int[] step, final int[] oShape) { 362 this.asView = asView; 363 this.start = start; 364 this.stop = stop; 365 this.step = step; 366 this.oShape = oShape; 367 oSize = ShapeUtils.calcLongSize(oShape); 368 } 369 370 @Override 371 public Object processField(Field field, Object o) { 372 return o; 373 } 374 375 @Override 376 public Class<? extends Annotation> getAnnClass() { 377 return Sliceable.class; 378 } 379 380 @Override 381 public int change(int axis) { 382 return 0; 383 } 384 385 @Override 386 public int getNewRank() { 387 return -1; 388 } 389 390 @Override 391 public ILazyDataset run(ILazyDataset lz) { 392 int rank = lz.getRank(); 393 if (start.length != rank) { 394 throw new IllegalArgumentException("Slice dimensions do not match dataset!"); 395 } 396 397 int[] shape = lz.getShape(); 398 int[] stt; 399 int[] stp; 400 int[] ste; 401 if (lz.getSize() == oSize) { 402 stt = start; 403 stp = stop; 404 ste = step; 405 } else { 406 stt = start.clone(); 407 stp = stop.clone(); 408 ste = step.clone(); 409 for (int i = 0; i < rank; i++) { 410 if (shape[i] >= oShape[i]) continue; 411 if (shape[i] == 1) { 412 stt[i] = 0; 413 stp[i] = 1; 414 ste[1] = 1; 415 } else { 416 throw new IllegalArgumentException("Sliceable dataset has invalid size!"); 417 } 418 } 419 } 420 421 if (asView || (lz instanceof IDataset)) 422 return lz.getSliceView(stt, stp, ste); 423 try { 424 return lz.getSlice(stt, stp, ste); 425 } catch (DatasetException e) { 426 logger.error("Could not slice dataset in metadata", e); 427 return null; 428 } 429 } 430 } 431 432 class MdsReshape implements MetadatasetAnnotationOperation { 433 private boolean matchRank; 434 private int[] oldShape; 435 private int[] newShape; 436 boolean onesOnly; 437 int[] differences; 438 439 /* 440 * if only ones then record differences (insertions and deletions) 441 * 442 * if shape changing, find broadcasted dimensions and disallow 443 * merging that include those dimensions 444 */ 445 public MdsReshape(final int[] oldShape, final int[] newShape) { 446 this.oldShape = oldShape; 447 this.newShape = newShape; 448 differences = null; 449 } 450 451 @Override 452 public Object processField(Field field, Object o) { 453 Annotation a = field.getAnnotation(Reshapeable.class); 454 if (a != null) { // cannot be null 455 matchRank = ((Reshapeable) a).matchRank(); 456 } 457 return o; 458 } 459 460 @Override 461 public Class<? extends Annotation> getAnnClass() { 462 return Reshapeable.class; 463 } 464 465 @Override 466 public int change(int axis) { 467 if (matchRank) { 468 if (differences == null) 469 init(); 470 471 if (onesOnly) { 472 return differences[axis]; 473 } 474 throw new UnsupportedOperationException("TODO support other shape operations"); 475 } 476 return 0; 477 } 478 479 @Override 480 public int getNewRank() { 481 return matchRank ? newShape.length : -1; 482 } 483 484 private void init() { 485 int or = oldShape.length - 1; 486 int nr = newShape.length - 1; 487 if (or < 0 || nr < 0) { // zero-rank shapes 488 onesOnly = true; 489 differences = new int[1]; 490 differences[0] = or < 0 ? nr + 1 : or + 1; 491 return; 492 } 493 int ob = 0; 494 int nb = 0; 495 onesOnly = true; 496 do { 497 while (oldShape[ob] == 1 && ob < or) { 498 ob++; // next non-unit dimension 499 } 500 while (newShape[nb] == 1 && nb < nr) { 501 nb++; 502 } 503 if (oldShape[ob++] != newShape[nb++]) { 504 onesOnly = false; 505 break; 506 } 507 } while (ob <= or && nb <= nr); 508 509 ob = 0; 510 nb = 0; 511 differences = new int[or + 2]; 512 if (onesOnly) { 513 // work out unit dimensions removed from or add to old 514 int j = 0; 515 do { 516 if (oldShape[ob] != 1 && newShape[nb] != 1) { 517 ob++; 518 nb++; 519 } else { 520 while (oldShape[ob] == 1 && ob < or) { 521 ob++; 522 differences[j]--; 523 } 524 while (newShape[nb] == 1 && nb < nr) { 525 nb++; 526 differences[j]++; 527 } 528 } 529 j++; 530 } while (ob <= or && nb <= nr && j <= or); 531 while (ob <= or && oldShape[ob] == 1) { 532 ob++; 533 differences[j]--; 534 } 535 while (nb <= nr && newShape[nb] == 1) { 536 nb++; 537 differences[j]++; 538 } 539 } else { 540 if (matchRank) { 541 logger.error("Combining dimensions is currently not supported"); 542 throw new IllegalArgumentException("Combining dimensions is currently not supported"); 543 } 544 // work out mapping: contiguous dimensions can be grouped or split 545 while (ob <= or && nb <= nr) { 546 int ol = oldShape[ob]; 547 while (ol == 1 && ol <= or) { 548 ob++; 549 ol = oldShape[ob]; 550 } 551 int oe = ob + 1; 552 int nl = newShape[nb]; 553 while (nl == 1 && nl <= nr) { 554 nb++; 555 nl = newShape[nb]; 556 } 557 int ne = nb + 1; 558 if (ol < nl) { 559 differences[ob] = 1; 560 do { // case where new shape combines several dimensions into one dimension 561 if (oe == (or + 1)) { 562 break; 563 } 564 differences[oe] = 1; 565 ol *= oldShape[oe++]; 566 } while (ol < nl); 567 differences[oe - 1] = oe - ob; // signal end with difference 568 if (nl != ol) { 569 logger.error("Single dimension is incompatible with subshape"); 570 throw new IllegalArgumentException("Single dimension is incompatible with subshape"); 571 } 572 } else if (ol > nl) { 573 do { // case where new shape spreads single dimension over several dimensions 574 if (ne == (nr + 1)) { 575 break; 576 } 577 nl *= newShape[ne++]; 578 } while (nl < ol); 579 if (nl != ol) { 580 logger.error("Subshape is incompatible with single dimension"); 581 throw new IllegalArgumentException("Subshape is incompatible with single dimension"); 582 } 583 584 } 585 586 ob = oe; 587 nb = ne; 588 } 589 590 } 591 } 592 593 @Override 594 public ILazyDataset run(ILazyDataset lz) { 595 if (differences == null) 596 init(); 597 598 int[] lshape = lz.getShape(); 599 if (Arrays.equals(newShape, lshape)) { 600 return lz; 601 } 602 int or = lz.getRank(); 603 int nr = newShape.length; 604 int[] nshape = new int[nr]; 605 Arrays.fill(nshape, 1); 606 if (onesOnly) { 607 // ignore omit removed dimensions 608 for (int i = 0, si = 0, di = 0; i < (or+1) && si <= or && di < nr; i++) { 609 int c = differences[i]; 610 if (c == 0) { 611 nshape[di++] = lshape[si++]; 612 } else if (c > 0) { 613 while (c-- > 0 && di < nr) { 614 di++; 615 } 616 } else if (c < 0) { 617 si -= c; // remove dimensions by skipping forward in source array 618 } 619 } 620 } else { 621 boolean[] broadcast = new boolean[or]; 622 for (int ob = 0; ob < or; ob++) { 623 broadcast[ob] = oldShape[ob] != 1 && lshape[ob] == 1; 624 } 625 int osize = lz.getSize(); 626 627 // cannot do 3x5x... to 15x... if metadata is broadcasting (i.e. 1x5x...) 628 int ob = 0; 629 int nsize = 1; 630 for (int i = 0; i < nr; i++) { 631 if (ob < or && broadcast[ob]) { 632 if (differences[ob] != 0) { 633 logger.error("Metadata contains a broadcast axis which cannot be reshaped"); 634 throw new IllegalArgumentException("Metadata contains a broadcast axis which cannot be reshaped"); 635 } 636 } else { 637 nshape[i] = nsize < osize ? newShape[i] : 1; 638 } 639 nsize *= nshape[i]; 640 ob++; 641 } 642 } 643 644 ILazyDataset nlz = lz.getSliceView(); 645 if (lz instanceof Dataset) { 646 nlz = ((Dataset) lz).reshape(nshape); 647 } else { 648 nlz = lz.getSliceView(); 649 nlz.setShape(nshape); 650 } 651 return nlz; 652 } 653 } 654 655 class MdsTranspose implements MetadatasetAnnotationOperation { 656 int[] map; 657 658 public MdsTranspose(final int[] axesMap) { 659 map = axesMap; 660 } 661 662 @SuppressWarnings({ "rawtypes", "unchecked" }) 663 @Override 664 public Object processField(Field f, Object o) { 665 // reorder arrays and lists according the axes map 666 if (o.getClass().isArray()) { 667 int l = Array.getLength(o); 668 if (l == map.length) { 669 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 670 for (int i = 0; i < l; i++) { 671 Array.set(narray, i, Array.get(o, map[i])); 672 } 673 for (int i = 0; i < l; i++) { 674 Array.set(o, i, Array.get(narray, i)); 675 } 676 } 677 } else if (o instanceof List<?>) { 678 List list = (List) o; 679 int l = list.size(); 680 if (l == map.length) { 681 Object narray = Array.newInstance(o.getClass().getComponentType(), l); 682 for (int i = 0; i < l; i++) { 683 Array.set(narray, i, list.get(map[i])); 684 } 685 list.clear(); 686 for (int i = 0; i < l; i++) { 687 list.add(Array.get(narray, i)); 688 } 689 } 690 } 691 return o; 692 } 693 694 @Override 695 public Class<? extends Annotation> getAnnClass() { 696 return Transposable.class; 697 } 698 699 @Override 700 public int change(int axis) { 701 return 0; 702 } 703 704 @Override 705 public int getNewRank() { 706 return -1; 707 } 708 709 @Override 710 public ILazyDataset run(ILazyDataset lz) { 711 return lz.getTransposedView(map); 712 } 713 } 714 715 class MdsDirty implements MetadatasetAnnotationOperation { 716 717 @Override 718 public Object processField(Field f, Object o) { 719 // throw exception if not boolean??? 720 Class<?> t = f.getType(); 721 if (t.equals(boolean.class) || t.equals(Boolean.class)) { 722 if (o.equals(false)) { 723 o = true; 724 } 725 } 726 return o; 727 } 728 729 @Override 730 public Class<? extends Annotation> getAnnClass() { 731 return Dirtiable.class; 732 } 733 734 @Override 735 public int change(int axis) { 736 return 0; 737 } 738 739 @Override 740 public int getNewRank() { 741 return -1; 742 } 743 744 @Override 745 public ILazyDataset run(ILazyDataset lz) { 746 return lz; 747 } 748 } 749 750 /** 751 * Slice all datasets in metadata that are annotated by @Sliceable. Call this on the new sliced 752 * dataset after cloning the metadata 753 * @param asView if true then just a view 754 * @param slice 755 */ 756 protected void sliceMetadata(boolean asView, final SliceND slice) { 757 processAnnotatedMetadata(new MdsSlice(asView, slice.getStart(), slice.getStop(), slice.getStep(), slice.getSourceShape()), true); 758 } 759 760 /** 761 * Reshape all datasets in metadata that are annotated by @Reshapeable. Call this when squeezing 762 * or setting the shape 763 * 764 * @param newShape 765 */ 766 protected void reshapeMetadata(final int[] oldShape, final int[] newShape) { 767 processAnnotatedMetadata(new MdsReshape(oldShape, newShape), true); 768 } 769 770 /** 771 * Transpose all datasets in metadata that are annotated by @Transposable. Call this on the transposed 772 * dataset after cloning the metadata 773 * @param axesMap 774 */ 775 protected void transposeMetadata(final int[] axesMap) { 776 processAnnotatedMetadata(new MdsTranspose(axesMap), true); 777 } 778 779 /** 780 * Dirty metadata that are annotated by @Dirtiable. Call this when the dataset has been modified 781 * @since 2.0 782 */ 783 protected void dirtyMetadata() { 784 processAnnotatedMetadata(new MdsDirty(), true); 785 } 786 787 @SuppressWarnings("unchecked") 788 private void processAnnotatedMetadata(MetadatasetAnnotationOperation op, boolean throwException) { 789 if (metadata == null) 790 return; 791 792 for (Class<? extends MetadataType> c : metadata.keySet()) { 793 for (MetadataType m : metadata.get(c)) { 794 if (m == null) 795 continue; 796 797 Class<? extends MetadataType> mc = m.getClass(); 798 do { // iterate over super-classes 799 processClass(op, m, mc, throwException); 800 Class<?> sclazz = mc.getSuperclass(); 801 if (!MetadataType.class.isAssignableFrom(sclazz)) 802 break; 803 mc = (Class<? extends MetadataType>) sclazz; 804 } while (true); 805 } 806 } 807 } 808 809 @SuppressWarnings({ "unchecked", "rawtypes" }) 810 private static void processClass(MetadatasetAnnotationOperation op, MetadataType m, Class<? extends MetadataType> mc, boolean throwException) { 811 for (Field f : mc.getDeclaredFields()) { 812 if (!f.isAnnotationPresent(op.getAnnClass())) 813 continue; 814 815 try { 816 f.setAccessible(true); 817 Object o = f.get(m); 818 if (o == null) 819 continue; 820 821 Object no = op.processField(f, o); 822 if (no != o) { 823 f.set(m, no); 824 continue; 825 } 826 Object r = null; 827 if (o instanceof ILazyDataset) { 828 try { 829 f.set(m, op.run((ILazyDataset) o)); 830 } catch (Exception e) { 831 logger.error("Problem processing " + o, e); 832 if (!catchExceptions) 833 throw e; 834 } 835 } else if (o.getClass().isArray()) { 836 int l = Array.getLength(o); 837 if (l <= 0) 838 continue; 839 840 for (int i = 0; r == null && i < l; i++) { 841 r = Array.get(o, i); 842 } 843 int n = op.getNewRank(); 844 if (r == null) { 845 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 846 f.set(m, Array.newInstance(o.getClass().getComponentType(), n < 0 ? l : n)); 847 } 848 continue; 849 } 850 if (n < 0) 851 n = l; 852 Object narray = Array.newInstance(r.getClass(), n); 853 for (int i = 0, si = 0, di = 0; di < n && si < l; i++) { 854 int c = op.change(i); 855 if (c == 0) { 856 Array.set(narray, di++, processObject(op, Array.get(o, si++))); 857 } else if (c > 0) { 858 di += c; // add nulls by skipping forward in destination array 859 } else if (c < 0) { 860 si -= c; // remove dimensions by skipping forward in source array 861 } 862 } 863 if (n == l) { 864 for (int i = 0; i < l; i++) { 865 Array.set(o, i, Array.get(narray, i)); 866 } 867 } else { 868 f.set(m, narray); 869 } 870 } else if (o instanceof List<?>) { 871 List list = (List) o; 872 int l = list.size(); 873 if (l <= 0) 874 continue; 875 876 for (int i = 0; r == null && i < l; i++) { 877 r = list.get(i); 878 } 879 int n = op.getNewRank(); 880 if (r == null) { 881 if (n < 0 || n != l) { // all nulls be need to match rank as necessary 882 list.clear(); 883 for (int i = 0, imax = n < 0 ? l : n; i < imax; i++) { 884 list.add(null); 885 } 886 } 887 continue; 888 } 889 890 if (n < 0) 891 n = l; 892 Object narray = Array.newInstance(r.getClass(), n); 893 for (int i = 0, si = 0, di = 0; i < l && si < l; i++) { 894 int c = op.change(i); 895 if (c == 0) { 896 Array.set(narray, di++, processObject(op, list.get(si++))); 897 } else if (c > 0) { 898 di += c; // add nulls by skipping forward in destination array 899 } else if (c < 0) { 900 si -= c; // remove dimensions by skipping forward in source array 901 } 902 } 903 list.clear(); 904 for (int i = 0; i < n; i++) { 905 list.add(Array.get(narray, i)); 906 } 907 } else if (o instanceof Map<?,?>) { 908 Map map = (Map) o; 909 for (Object k : map.keySet()) { 910 map.put(k, processObject(op, map.get(k))); 911 } 912 } 913 } catch (Exception e) { 914 logger.error("Problem occurred when processing metadata of class {}: {}", mc.getCanonicalName(), e); 915 if (throwException) 916 throw new RuntimeException(e); 917 } 918 } 919 } 920 921 @SuppressWarnings({ "unchecked", "rawtypes" }) 922 private static Object processObject(MetadatasetAnnotationOperation op, Object o) throws Exception { 923 if (o == null) 924 return o; 925 926 if (o instanceof ILazyDataset) { 927 try { 928 return op.run((ILazyDataset) o); 929 } catch (Exception e) { 930 logger.error("Problem processing " + o, e); 931 if (!catchExceptions) 932 throw e; 933 } 934 } else if (o.getClass().isArray()) { 935 int l = Array.getLength(o); 936 for (int i = 0; i < l; i++) { 937 Array.set(o, i, processObject(op, Array.get(o, i))); 938 } 939 } else if (o instanceof List<?>) { 940 List list = (List) o; 941 for (int i = 0, imax = list.size(); i < imax; i++) { 942 list.set(i, processObject(op, list.get(i))); 943 } 944 } else if (o instanceof Map<?,?>) { 945 Map map = (Map) o; 946 for (Object k : map.keySet()) { 947 map.put(k, processObject(op, map.get(k))); 948 } 949 } 950 return o; 951 } 952 953 protected void restoreMetadata(Map<Class<? extends MetadataType>, List<MetadataType>> oldMetadata) { 954 for (Class<? extends MetadataType> mc : oldMetadata.keySet()) { 955 metadata.put(mc, oldMetadata.get(mc)); 956 } 957 } 958 959 @SuppressWarnings("deprecation") 960 protected ILazyDataset createFromSerializable(Serializable blob, boolean keepLazy) { 961 ILazyDataset d = null; 962 if (blob instanceof ILazyDataset) { 963 d = (ILazyDataset) blob; 964 if (d instanceof IDataset) { 965 Dataset ed = DatasetUtils.convertToDataset((IDataset) d); 966 int is = ed.getElementsPerItem(); 967 if (is != 1 && is != getElementsPerItem()) { 968 throw new IllegalArgumentException("Dataset has incompatible number of elements with this dataset"); 969 } 970 d = ed.cast(is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 971 } else if (!keepLazy) { 972 final int is = getElementsPerItem(); 973 try { 974 d = DatasetUtils.cast(d.getSlice(), is == 1 ? Dataset.FLOAT64 : Dataset.ARRAYFLOAT64); 975 } catch (DatasetException e) { 976 logger.error("Could not get data from lazy dataset", e); 977 return null; 978 } 979 } 980 } else { 981 final int is = getElementsPerItem(); 982 if (is == 1) { 983 d = DatasetFactory.createFromObject(Dataset.FLOAT64, blob); 984 } else { 985 try { 986 d = DatasetFactory.createFromObject(is, Dataset.ARRAYFLOAT64, blob); 987 } catch (IllegalArgumentException e) { // if only single value supplied try again 988 d = DatasetFactory.createFromObject(Dataset.FLOAT64, blob); 989 } 990 } 991 if (d.getSize() == getSize() && !Arrays.equals(d.getShape(), shape)) { 992 d.setShape(shape.clone()); 993 } 994 } 995 List<int[]> s = BroadcastUtils.broadcastShapesToMax(shape, d.getShape()); 996 d.setShape(s.get(0)); 997 998 return d; 999 } 1000 1001 @Override 1002 public void setErrors(Serializable errors) { 1003 if (shape == null) { 1004 throw new IllegalArgumentException("Cannot set errors for null dataset"); 1005 } 1006 if (errors == null) { 1007 clearMetadata(ErrorMetadata.class); 1008 return; 1009 } 1010 if (errors == this) { 1011 logger.warn("Ignoring setting error to itself as this will lead to infinite recursion"); 1012 return; 1013 } 1014 1015 ILazyDataset errorData = createFromSerializable(errors, true); 1016 1017 ErrorMetadata emd = getErrorMetadata(); 1018 if (emd == null) { 1019 try { 1020 emd = MetadataFactory.createMetadata(ErrorMetadata.class); 1021 setMetadata(emd); 1022 } catch (MetadataException me) { 1023 logger.error("Could not create metadata", me); 1024 } 1025 } 1026 emd.setError(errorData); 1027 } 1028 1029 protected ErrorMetadata getErrorMetadata() { 1030 try { 1031 List<ErrorMetadata> el = getMetadata(ErrorMetadata.class); 1032 if (el != null && !el.isEmpty()) { 1033 return el.get(0); 1034 } 1035 } catch (Exception e) { 1036 } 1037 return null; 1038 } 1039 1040 @Override 1041 public ILazyDataset getErrors() { 1042 ErrorMetadata emd = getErrorMetadata(); 1043 return emd == null ? null : emd.getError(); 1044 } 1045 1046 @Override 1047 public boolean hasErrors() { 1048 return LazyDatasetBase.this.getErrors() != null; 1049 } 1050 1051 /** 1052 * Check permutation axes 1053 * @param shape 1054 * @param axes 1055 * @return cleaned up axes or null if trivial 1056 */ 1057 public static int[] checkPermutatedAxes(int[] shape, int... axes) { 1058 int rank = shape.length; 1059 1060 if (axes == null || axes.length == 0) { 1061 axes = new int[rank]; 1062 for (int i = 0; i < rank; i++) { 1063 axes[i] = rank - 1 - i; 1064 } 1065 } 1066 1067 if (axes.length != rank) { 1068 logger.error("axis permutation has length {} that does not match dataset's rank {}", axes.length, rank); 1069 throw new IllegalArgumentException("axis permutation does not match shape of dataset"); 1070 } 1071 1072 // check all permutation values are within bounds 1073 for (int d : axes) { 1074 if (d < 0 || d >= rank) { 1075 logger.error("axis permutation contains element {} outside rank of dataset", d); 1076 throw new IllegalArgumentException("axis permutation contains element outside rank of dataset"); 1077 } 1078 } 1079 1080 // check for a valid permutation (is this an unnecessary restriction?) 1081 int[] perm = axes.clone(); 1082 Arrays.sort(perm); 1083 1084 for (int i = 0; i < rank; i++) { 1085 if (perm[i] != i) { 1086 logger.error("axis permutation is not valid: it does not contain complete set of axes"); 1087 throw new IllegalArgumentException("axis permutation does not contain complete set of axes"); 1088 } 1089 } 1090 1091 if (Arrays.equals(axes, perm)) 1092 return null; // signal identity or trivial permutation 1093 1094 return axes; 1095 } 1096}