001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.util.Arrays;
016
017import org.eclipse.january.DatasetException;
018import org.eclipse.january.IMonitor;
019import org.eclipse.january.metadata.StatisticsMetadata;
020import org.eclipse.january.metadata.internal.StatisticsMetadataImpl;
021import org.slf4j.Logger;
022import org.slf4j.LoggerFactory;
023
024/**
025 * Generic container class for data that is compound in nature
026 * 
027 * Each subclass has an array of compound types, items of this array are composed of primitive types
028 * 
029 * Data items can be Complex, Vector, etc
030 * 
031 */
032public abstract class AbstractCompoundDataset extends AbstractDataset implements CompoundDataset {
033        // pin UID to base class
034        private static final long serialVersionUID = Dataset.serialVersionUID;
035
036        private static final Logger logger = LoggerFactory.getLogger(AbstractCompoundDataset.class);
037
038        protected int isize; // number of elements per item
039
040        @Override
041        public int getElementsPerItem() {
042                return isize;
043        }
044
045        @Override
046        protected int get1DIndex(final int i) {
047                int n = super.get1DIndex(i);
048                return stride == null ? isize * n : n;
049        }
050
051        @Override
052        protected int get1DIndex(final int i, final int j) {
053                int n = super.get1DIndex(i, j);
054                return stride == null ? isize * n : n;
055        }
056
057        @Override
058        protected int get1DIndexFromShape(final int[] n) {
059                return isize * super.get1DIndexFromShape(n);
060        }
061
062        @Override
063        public Dataset getUniqueItems() {
064                throw new UnsupportedOperationException("Cannot sort compound datasets");
065        }
066
067        @Override
068        public IndexIterator getIterator(final boolean withPosition) {
069                if (stride != null) {
070                        return base.getSize() == 1 ? 
071                                        (withPosition ? new PositionIterator(offset, shape) :  new SingleItemIterator(offset, size)) : new StrideIterator(isize, shape, stride, offset);
072                }
073                return withPosition ? getSliceIterator(null, null, null) :
074                        new ContiguousIterator(size, isize);
075        }
076
077        /**
078         * Get an iterator that picks out the chosen element from all items
079         * @param element to choose
080         * @return an iterator
081         */
082        public IndexIterator getIterator(int element) {
083                if (element < 0)
084                        element += isize;
085                if (element < 0 || element > isize) {
086                        logger.error("Invalid choice of element: {}/{}", element, isize);
087                        throw new IllegalArgumentException("Invalid choice of element: " + element + "/" + isize);
088                }
089
090                final IndexIterator it;
091                if (stride != null) {
092                        it = base.getSize() == 1 ? new SingleItemIterator(offset + element, size) : new StrideIterator(isize, shape, stride, offset, element);
093                } else {
094                        it = new ContiguousIterator(size, isize, element);
095                }
096
097                return it;
098        }
099
100        /**
101         * @param slice to define iterator
102         * @return an slice iterator that operates like an IndexIterator
103         */
104        @Override
105        protected IndexIterator internalGetSliceIterator(SliceND slice) {
106                if (ShapeUtils.calcLongSize(slice.getShape()) == 0) {
107                        return new NullIterator(shape, slice.getShape());
108                }
109                if (stride != null) {
110                        return new StrideIterator(isize, shape, stride, offset, slice);
111                }
112
113                return new SliceIterator(shape, size, isize, slice);
114        }
115
116        /**
117         * Constructor required for serialisation.
118         */
119        public AbstractCompoundDataset() {
120        }
121
122        @Override
123        public boolean equals(Object obj) {
124                if (!super.equals(obj)) {
125                        return false;
126                }
127
128                CompoundDataset other = (CompoundDataset) obj;
129                return isize == other.getElementsPerItem();
130        }
131
132        @Override
133        public int hashCode() {
134                return getCompoundStats().getHash(shape);
135        }
136
137        @Override
138        public CompoundDataset cast(boolean repeat, int dtype, int isize) {
139                return (CompoundDataset) super.cast(repeat, dtype, isize);
140        }
141
142        @Override
143        public CompoundDataset cast(int dtype) {
144                return (CompoundDataset) super.cast(dtype);
145        }
146
147        @Override
148        abstract public AbstractCompoundDataset clone();
149
150        @Override
151        public CompoundDataset flatten() {
152                return (CompoundDataset) super.flatten();
153        }
154
155        @Override
156        public CompoundDataset getBy1DIndex(IntegerDataset index) {
157                return (CompoundDataset) super.getBy1DIndex(index);
158        }
159
160        @Override
161        public CompoundDataset getByBoolean(Dataset selection) {
162                return (CompoundDataset) super.getByBoolean(selection);
163        }
164
165        @Override
166        public CompoundDataset getByIndexes(Object... indexes) {
167                return (CompoundDataset) super.getByIndexes(indexes);
168        }
169
170        @Override
171        public CompoundDataset getSlice(IMonitor mon, int[] start, int[] stop, int[] step) {
172                return (CompoundDataset) super.getSlice(mon, start, stop, step);
173        }
174
175        @Override
176        public CompoundDataset getSlice(IMonitor mon, Slice... slice) {
177                return (CompoundDataset) super.getSlice(mon, slice);
178        }
179
180        @Override
181        public CompoundDataset getSlice(IMonitor mon, SliceND slice) {
182                return (CompoundDataset) super.getSlice(mon, slice);
183        }
184
185        @Override
186        public CompoundDataset getSlice(int[] start, int[] stop, int[] step) {
187                return (CompoundDataset) super.getSlice(start, stop, step);
188        }
189
190        @Override
191        public CompoundDataset getSlice(Slice... slice) {
192                return (CompoundDataset) super.getSlice(slice);
193        }
194
195        @Override
196        public CompoundDataset getSlice(SliceND slice) {
197                return (CompoundDataset) super.getSlice(slice);
198        }
199
200        @Override
201        abstract public AbstractCompoundDataset getSlice(SliceIterator iterator);
202
203        @Override
204        public CompoundDataset getSliceView(int[] start, int[] stop, int[] step) {
205                return (CompoundDataset) super.getSliceView(start, stop, step);
206        }
207
208        @Override
209        public CompoundDataset getSliceView(Slice... slice) {
210                return (CompoundDataset) super.getSliceView(slice);
211        }
212
213        @Override
214        public CompoundDataset getSliceView(SliceND slice) {
215                return (CompoundDataset) super.getSliceView(slice);
216        }
217
218        @Override
219        public CompoundDataset getTransposedView(int... axes) {
220                return (CompoundDataset) super.getTransposedView(axes);
221        }
222
223        @Override
224        abstract public AbstractCompoundDataset getView(boolean deepCopyMetadata);
225
226        @Override
227        public CompoundDataset getBroadcastView(int... broadcastShape) {
228                return (CompoundDataset) super.getBroadcastView(broadcastShape);
229        }
230
231        @Override
232        public CompoundDataset ifloorDivide(Object o) {
233                return (CompoundDataset) super.ifloorDivide(o);
234        }
235
236        @Override
237        public CompoundDataset reshape(int... shape) {
238                return (CompoundDataset) super.reshape(shape);
239        }
240
241        @Override
242        public CompoundDataset setSlice(Object obj, int[] start, int[] stop, int[] step) {
243                return (CompoundDataset) super.setSlice(obj, start, stop, step);
244        }
245
246        @Override
247        public CompoundDataset setSlice(Object object, Slice... slice) {
248                return (CompoundDataset) super.setSlice(object, slice);
249        }
250
251        @Override
252        public CompoundDataset sort(Integer axis) {
253                throw new UnsupportedOperationException("Cannot sort dataset");
254        }
255
256        @Override
257        public CompoundDataset squeezeEnds() {
258                return (CompoundDataset) super.squeezeEnds();
259        }
260
261        @Override
262        public CompoundDataset squeeze() {
263                return (CompoundDataset) super.squeeze();
264        }
265
266        @Override
267        public CompoundDataset squeeze(boolean onlyFromEnd) {
268                return (CompoundDataset) super.squeeze(onlyFromEnd);
269        }
270
271        @Override
272        public CompoundDataset swapAxes(int axis1, int axis2) {
273                return (CompoundDataset) super.swapAxes(axis1, axis2);
274        }
275
276        @Override
277        public synchronized CompoundDataset synchronizedCopy() {
278                return clone();
279        }
280
281        @Override
282        public CompoundDataset transpose(int... axes) {
283                return (CompoundDataset) super.transpose(axes);
284        }
285
286        /**
287         * @since 2.0
288         * @return first value
289         */
290        abstract protected double getFirstValue();
291
292        abstract protected double getFirstValue(final int i);
293
294        abstract protected double getFirstValue(final int i, final int j);
295
296        abstract protected double getFirstValue(final int...pos);
297
298        @Override
299        public boolean getBoolean() {
300                return getFirstValue() != 0;
301        }
302
303        @Override
304        public boolean getBoolean(final int i) {
305                return getFirstValue(i) != 0;
306        }
307
308        @Override
309        public boolean getBoolean(final int i, final int j) {
310                return getFirstValue(i, j) != 0;
311        }
312
313        @Override
314        public boolean getBoolean(final int... pos) {
315                return getFirstValue(pos) != 0;
316        }
317
318        @Override
319        public byte getByte() {
320                return (byte) getFirstValue();
321        }
322
323        @Override
324        public byte getByte(final int i) {
325                return (byte) getFirstValue(i);
326        }
327
328        @Override
329        public byte getByte(final int i, final int j) {
330                return (byte) getFirstValue(i, j);
331        }
332
333        @Override
334        public byte getByte(final int... pos) {
335                return (byte) getFirstValue(pos);
336        }
337
338        @Override
339        public short getShort() {
340                return (short) getFirstValue();
341        }
342
343        @Override
344        public short getShort(final int i) {
345                return (short) getFirstValue(i);
346        }
347
348        @Override
349        public short getShort(final int i, final int j) {
350                return (short) getFirstValue(i, j);
351        }
352
353        @Override
354        public short getShort(final int... pos) {
355                return (short) getFirstValue(pos);
356        }
357
358        @Override
359        public int getInt() {
360                return (int) getFirstValue();
361        }
362
363        @Override
364        public int getInt(final int i) {
365                return (int) getFirstValue(i);
366        }
367
368        @Override
369        public int getInt(final int i, final int j) {
370                return (int) getFirstValue(i, j);
371        }
372
373        @Override
374        public int getInt(final int... pos) {
375                return (int) getFirstValue(pos);
376        }
377
378        @Override
379        public long getLong() {
380                return (long) getFirstValue();
381        }
382
383        @Override
384        public long getLong(final int i) {
385                return (long) getFirstValue(i);
386        }
387
388        @Override
389        public long getLong(final int i, final int j) {
390                return (long) getFirstValue(i, j);
391        }
392
393        @Override
394        public long getLong(final int... pos) {
395                return (long) getFirstValue(pos);
396        }
397
398        @Override
399        public float getFloat() {
400                return (float) getFirstValue();
401        }
402
403        @Override
404        public float getFloat(final int i) {
405                return (float) getFirstValue(i);
406        }
407
408        @Override
409        public float getFloat(final int i, final int j) {
410                return (float) getFirstValue(i, j);
411        }
412
413        @Override
414        public float getFloat(final int... pos) {
415                return (float) getFirstValue(pos);
416        }
417
418        @Override
419        public double getDouble() {
420                return getFirstValue();
421        }
422
423        @Override
424        public double getDouble(final int i) {
425                return getFirstValue(i);
426        }
427
428        @Override
429        public double getDouble(final int i, final int j) {
430                return getFirstValue(i, j);
431        }
432
433        @Override
434        public double getDouble(final int... pos) {
435                return getFirstValue(pos);
436        }
437
438        @Override
439        public void getDoubleArray(final double[] darray) {
440                getDoubleArrayAbs(getFirst1DIndex(), darray);
441        }
442
443        @Override
444        public void getDoubleArray(final double[] darray, final int i) {
445                getDoubleArrayAbs(get1DIndex(i), darray);
446        }
447
448        @Override
449        public void getDoubleArray(final double[] darray, final int i, final int j) {
450                getDoubleArrayAbs(get1DIndex(i, j), darray);
451        }
452
453        @Override
454        public void getDoubleArray(final double[] darray, final int... pos) {
455                getDoubleArrayAbs(get1DIndex(pos), darray);
456        }
457
458        /**
459         * @return statistics metadata
460         * @since 2.0
461         */
462        @SuppressWarnings("unchecked")
463        protected StatisticsMetadata<double[]> getCompoundStats() {
464                StatisticsMetadata<double[]> md = getFirstMetadata(StatisticsMetadata.class);
465                if (md == null || md.isDirty(this)) {
466                        md = new StatisticsMetadataImpl<double[]>();
467                        md.initialize(this);
468                        setMetadata(md);
469                }
470                return md;
471        }
472
473        @Override
474        public IntegerDataset argMax(int axis, boolean... ignoreInvalids) {
475                logger.error("Cannot compare compound numbers");
476                throw new UnsupportedOperationException("Cannot compare compound numbers");
477        }
478
479        @Override
480        public IntegerDataset argMin(int axis, boolean... ignoreInvalids) {
481                logger.error("Cannot compare compound numbers");
482                throw new UnsupportedOperationException("Cannot compare compound numbers");
483        }
484
485        @Override
486        public Number max(boolean... ignoreInvalids) {
487                logger.error("Cannot compare compound numbers");
488                throw new UnsupportedOperationException("Cannot compare compound numbers");
489        }
490
491        @Override
492        public CompoundDataset max(int axis, boolean... ignoreInvalids) {
493                logger.error("Cannot compare compound numbers");
494                throw new UnsupportedOperationException("Cannot compare compound numbers");
495        }
496
497        @Override
498        public Number min(boolean... ignoreInvalids) {
499                logger.error("Cannot compare compound numbers");
500                throw new UnsupportedOperationException("Cannot compare compound numbers");
501        }
502
503        @Override
504        public CompoundDataset min(int axis, boolean... ignoreInvalids) {
505                logger.error("Cannot compare compound numbers");
506                throw new UnsupportedOperationException("Cannot compare compound numbers");
507        }
508
509
510        @Override
511        public int[] maxPos(boolean... ignoreNaNs) {
512                logger.error("Cannot compare compound numbers");
513                throw new UnsupportedOperationException("Cannot compare compound numbers");
514        }
515
516        @Override
517        public int[] minPos(boolean... ignoreNaNs) {
518                logger.error("Cannot compare compound numbers");
519                throw new UnsupportedOperationException("Cannot compare compound numbers");
520        }
521
522        @Override
523        public CompoundDataset peakToPeak(int axis, boolean... ignoreInvalids) {
524                logger.error("Cannot compare compound numbers");
525                throw new UnsupportedOperationException("Cannot compare compound numbers");
526        }
527
528        @Override
529        public double[] maxItem() {
530                return getCompoundStats().getMaximum();
531        }
532
533        @Override
534        public double[] minItem() {
535                return getCompoundStats().getMinimum();
536        }
537
538        @Override
539        public Object mean(boolean... ignoreInvalids) {
540                return getCompoundStats().getMean();
541        }
542
543        @Override
544        public CompoundDataset mean(int axis, boolean... ignoreInvalids) {
545                return (CompoundDataset) super.mean(axis, ignoreInvalids);
546        }
547
548        @Override
549        public CompoundDataset product(int axis, boolean... ignoreInvalids) {
550                return (CompoundDataset) super.product(axis, ignoreInvalids);
551        }
552
553        @Override
554        public CompoundDataset rootMeanSquare(int axis, boolean... ignoreInvalids) {
555                return (CompoundDataset) super.rootMeanSquare(axis, ignoreInvalids);
556        }
557
558        @Override
559        public CompoundDataset stdDeviation(int axis) {
560                return (CompoundDataset) super.stdDeviation(axis, false);
561        }
562
563        @Override
564        public CompoundDataset stdDeviation(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
565                return (CompoundDataset) super.stdDeviation(axis, isWholePopulation, ignoreInvalids);
566        }
567
568        @Override
569        public Object sum(boolean... ignoreInvalids) {
570                return getCompoundStats().getSum();
571        }
572
573        @Override
574        public CompoundDataset sum(int axis, boolean... ignoreInvalids) {
575                return (CompoundDataset) super.sum(axis, ignoreInvalids);
576        }
577
578        @Override
579        public double variance(boolean isWholePopulation, boolean... ignoreInvalids) {
580                return getCompoundStats().getVariance(isWholePopulation, ignoreInvalids);
581        }
582
583        @Override
584        public CompoundDataset variance(int axis) {
585                return (CompoundDataset) super.variance(axis, false);
586        }
587
588        @Override
589        public CompoundDataset variance(int axis, boolean isWholePopulation, boolean... ignoreInvalids) {
590                return (CompoundDataset) super.variance(axis, isWholePopulation, ignoreInvalids);
591        }
592
593        @Override
594        public double rootMeanSquare(boolean... ignoreInvalids) {
595                StatisticsMetadata<double[]> stats = getCompoundStats();
596
597                double[] mean = stats.getMean(ignoreInvalids);
598                double result = 0;
599                for (int i = 0; i < isize; i++) {
600                        double m = mean[i];
601                        result += m * m;
602                }
603                return Math.sqrt(result + stats.getVariance(true));
604        }
605
606        /**
607         * @return error
608         */
609        private CompoundDataset getInternalError() {
610                ILazyDataset led = super.getErrors();
611                if (led == null)
612                        return null;
613
614                Dataset ed = null;
615                try {
616                        ed = DatasetUtils.sliceAndConvertLazyDataset(led);
617                } catch (DatasetException e) {
618                        logger.error("Could not get data from lazy dataset", e);
619                }
620
621                CompoundDataset ced; // ensure it has the same number of elements
622                if (!(ed instanceof CompoundDataset) || ed.getElementsPerItem() != isize) {
623                        ced = new CompoundDoubleDataset(isize, true, ed);
624                } else {
625                        ced = (CompoundDataset) ed;
626                }
627                
628                if (led != ced) {
629                        setErrors(ced); // set back
630                }
631                return ced;
632        }
633
634        @Override
635        public CompoundDataset getErrors() {
636                CompoundDataset ed = getInternalError();
637                if (ed == null)
638                        return null;
639
640                return ed.getBroadcastView(shape);
641        }
642
643        @Override
644        public double getError(final int i) {
645                return calcError(getInternalErrorArray(true, i));
646        }
647
648        @Override
649        public double getError(final int i, final int j) {
650                return calcError(getInternalErrorArray(true, i, j));
651        }
652
653        @Override
654        public double getError(final int... pos) {
655                return calcError(getInternalErrorArray(true, pos));
656        }
657
658        private double calcError(double[] es) {
659                if (es == null)
660                        return 0;
661
662                // assume elements are independent
663                double e = 0;
664                for (int k = 0; k < isize; k++) {
665                        e += es[k];
666                }
667
668                return Math.sqrt(e);
669        }
670
671        @Override
672        public double[] getErrorArray(final int i) {
673                return getInternalErrorArray(false, i);
674        }
675
676        @Override
677        public double[] getErrorArray(final int i, final int j) {
678                return getInternalErrorArray(false, i, j);
679        }
680
681        @Override
682        public double[] getErrorArray(final int... pos) {
683                return getInternalErrorArray(false, pos);
684        }
685
686        private Dataset getInternalError(final boolean squared) {
687                Dataset sed = squared ? getInternalSquaredError() : getInternalError();
688                if (sed == null)
689                        return null;
690
691                return sed.getBroadcastView(shape);
692        }
693
694        private double[] getInternalErrorArray(final boolean squared, final int i) {
695                Dataset sed = getInternalError(squared);
696                if (sed == null)
697                        return null;
698
699                double[] es;
700                if (sed instanceof CompoundDoubleDataset) {
701                        es = ((CompoundDoubleDataset) sed).getDoubleArray(i);
702                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
703                                Arrays.fill(es, es[0]);
704                        }
705                } else {
706                        es = new double[isize];
707                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(i));
708                }
709                return es;
710        }
711
712        private double[] getInternalErrorArray(final boolean squared, final int i, final int j) {
713                Dataset sed = getInternalError(squared);
714                if (sed == null)
715                        return null;
716
717                double[] es;
718                if (sed instanceof CompoundDoubleDataset) {
719                        es = ((CompoundDoubleDataset) sed).getDoubleArray(i, j);
720                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
721                                Arrays.fill(es, es[0]);
722                        }
723                } else {
724                        es = new double[isize];
725                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(i, j));
726                }
727                return es;
728        }
729
730        private double[] getInternalErrorArray(final boolean squared, final int... pos) {
731                Dataset sed = getInternalError(squared);
732                if (sed == null)
733                        return null;
734
735                double[] es = new double[isize];
736                if (sed instanceof CompoundDoubleDataset) {
737                        es = ((CompoundDoubleDataset) sed).getDoubleArray(pos);
738                        if (sed.getElementsPerItem() != isize) { // ensure error is broadcasted
739                                Arrays.fill(es, es[0]);
740                        }
741                } else {
742                        es = new double[isize];
743                        Arrays.fill(es, ((DoubleDataset) sed).getDouble(pos));
744                }
745                return es;
746        }
747}
748