001/*-
002 *******************************************************************************
003 * Copyright (c) 2011, 2016 Diamond Light Source Ltd.
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *    Peter Chang - initial API and implementation and/or initial documentation
011 *******************************************************************************/
012
013package org.eclipse.january.dataset;
014
015import java.io.Serializable;
016import java.lang.reflect.Array;
017import java.util.ArrayList;
018import java.util.Arrays;
019import java.util.Comparator;
020import java.util.List;
021
022import org.apache.commons.math3.util.MathArrays;
023import org.eclipse.january.DatasetException;
024import org.slf4j.Logger;
025import org.slf4j.LoggerFactory;
026
027/**
028 * Utilities for manipulating datasets
029 */
030public class DatasetUtils {
031
032        /**
033         * Setup the logging facilities
034         */
035        private static final Logger utilsLogger = LoggerFactory.getLogger(DatasetUtils.class);
036
037        /**
038         * Append copy of dataset with another dataset along n-th axis
039         *
040         * @param a first dataset
041         * @param b second dataset
042         * @param axis
043         *            number of axis (negative number counts from last)
044         * @return appended dataset
045         */
046        public static Dataset append(IDataset a, IDataset b, int axis) {
047                final int[] ashape = a.getShape();
048                final int rank = ashape.length;
049                final int[] bshape = b.getShape();
050                if (rank != bshape.length) {
051                        throw new IllegalArgumentException("Incompatible number of dimensions");
052                }
053                axis = ShapeUtils.checkAxis(rank, axis);
054
055                for (int i = 0; i < rank; i++) {
056                        if (i != axis && ashape[i] != bshape[i]) {
057                                throw new IllegalArgumentException("Incompatible dimensions");
058                        }
059                }
060                final int[] nshape = new int[rank];
061                for (int i = 0; i < rank; i++) {
062                        nshape[i] = ashape[i];
063                }
064                nshape[axis] += bshape[axis];
065                final Class<? extends Dataset> ot = InterfaceUtils.getInterface(b);
066                final Class<? extends Dataset> dt = InterfaceUtils.getInterface(a);
067                Dataset ds = DatasetFactory.zeros(a.getElementsPerItem(), InterfaceUtils.getBestInterface(ot, dt), nshape);
068                IndexIterator iter = ds.getIterator(true);
069                int[] pos = iter.getPos();
070                while (iter.hasNext()) {
071                        int d = ashape[axis];
072                        if (pos[axis] < d) {
073                                ds.setObjectAbs(iter.index, a.getObject(pos));
074                        } else {
075                                pos[axis] -= d;
076                                ds.setObjectAbs(iter.index, b.getObject(pos));
077                                pos[axis] += d;
078                        }
079                }
080
081                return ds;
082        }
083
084        /**
085         * Changes specific items of dataset by replacing them with other array
086         * @param <T> dataset class
087         * @param a dataset
088         * @param indices dataset interpreted as integers where to put items
089         * @param values to set
090         * @return changed dataset
091         */
092        public static <T extends Dataset> T put(final T a, final Dataset indices, Object values) {
093                IndexIterator it = indices.getIterator();
094                Dataset vd = DatasetFactory.createFromObject(values).flatten();
095                int vlen = vd.getSize();
096                int v = 0;
097                while (it.hasNext()) {
098                        if (v >= vlen) v -= vlen;
099
100                        a.setObjectAbs((int) indices.getElementLongAbs(it.index), vd.getObjectAbs(v++));
101                }
102                return a;
103        }
104
105        /**
106         * Changes specific items of dataset by replacing them with other array
107         * @param <T> dataset class
108         * @param a dataset
109         * @param indices where to put items
110         * @param values to set
111         * @return changed dataset
112         */
113        public static <T extends Dataset> T put(final T a, final int[] indices, Object values) {
114                int ilen = indices.length;
115                Dataset vd = DatasetFactory.createFromObject(values).flatten();
116                int vlen = vd.getSize();
117                for (int i = 0, v= 0; i < ilen; i++) {
118                        if (v >= vlen) v -= vlen;
119
120                        a.setObjectAbs(indices[i], vd.getObjectAbs(v++));
121                }
122                return a;
123        }
124
125        /**
126         * Take items from dataset along an axis
127         * @param <T> dataset class
128         * @param a dataset
129         * @param indices dataset interpreted as integers where to take items
130         * @param axis if null, then use flattened view
131         * @return a sub-dataset
132         */
133        public static <T extends Dataset> T take(final T a, final Dataset indices, Integer axis) {
134                IntegerDataset indexes = indices.flatten().cast(IntegerDataset.class);
135                return take(a, indexes.getData(), axis);
136        }
137
138        /**
139         * Take items from dataset along an axis
140         * @param <T> dataset class
141         * @param a dataset
142         * @param indices where to take items
143         * @param axis if null, then use flattened view
144         * @return a sub-dataset
145         */
146        public static <T extends Dataset> T take(final T a, final int[] indices, Integer axis) {
147                if (indices == null || indices.length == 0) {
148                        utilsLogger.error("No indices given");
149                        throw new IllegalArgumentException("No indices given");
150                }
151                int[] ashape = a.getShape();
152                final int rank = ashape.length;
153                final int ilen = indices.length;
154
155                T result;
156                if (axis == null) {
157                        ashape = new int[1];
158                        ashape[0] = ilen;
159                        result = DatasetFactory.zeros(a, ashape);
160                        Serializable src = a.getBuffer();
161                        for (int i = 0; i < ilen; i++) {
162                                ((AbstractDataset) result).setItemDirect(i, indices[i], src);
163                        }
164                } else {
165                        axis = a.checkAxis(axis);
166                        ashape[axis] = ilen;
167                        result = DatasetFactory.zeros(a, ashape);
168
169                        int[] dpos = new int[rank];
170                        int[] spos = new int[rank];
171                        boolean[] axes = new boolean[rank];
172                        Arrays.fill(axes, true);
173                        axes[axis] = false;
174                        Serializable src = a.getBuffer();
175                        for (int i = 0; i < ilen; i++) {
176                                spos[axis] = indices[i];
177                                dpos[axis] = i;
178                                SliceIterator siter = a.getSliceIteratorFromAxes(spos, axes);
179                                SliceIterator diter = result.getSliceIteratorFromAxes(dpos, axes);
180
181                                while (siter.hasNext() && diter.hasNext()) {
182                                        ((AbstractDataset) result).setItemDirect(diter.index, siter.index, src);
183                                }
184                        }
185                }
186                result.setDirty();
187                return result;
188        }
189
190        /**
191         * Construct a dataset that contains the original dataset repeated the number
192         * of times in each axis given by corresponding entries in the reps array
193         *
194         * @param a dataset
195         * @param reps repetitions in each dimension
196         * @return tiled dataset
197         */
198        public static Dataset tile(final IDataset a, int... reps) {
199                int[] shape = a.getShape();
200                int rank = shape.length;
201                final int rlen = reps.length;
202
203                // expand shape
204                if (rank < rlen) {
205                        int[] newShape = new int[rlen];
206                        int extraRank = rlen - rank;
207                        for (int i = 0; i < extraRank; i++) {
208                                newShape[i] = 1;
209                        }
210                        for (int i = 0; i < rank; i++) {
211                                newShape[i+extraRank] = shape[i];
212                        }
213
214                        shape = newShape;
215                        rank = rlen;
216                } else if (rank > rlen) {
217                        int[] newReps = new int[rank];
218                        int extraRank = rank - rlen;
219                        for (int i = 0; i < extraRank; i++) {
220                                newReps[i] = 1;
221                        }
222                        for (int i = 0; i < rlen; i++) {
223                                newReps[i+extraRank] = reps[i];
224                        }
225                        reps = newReps;
226                }
227
228                // calculate new shape
229                int[] newShape = new int[rank];
230                for (int i = 0; i < rank; i++) {
231                        newShape[i] = shape[i]*reps[i];
232                }
233
234                Dataset tdata = DatasetFactory.zeros(a.getElementsPerItem(), InterfaceUtils.getInterfaceFromClass(a.getElementsPerItem(), a.getElementClass()), newShape);
235
236                // decide which way to put slices
237                boolean manyColumns;
238                if (rank == 1)
239                        manyColumns = true;
240                else
241                        manyColumns = shape[rank-1] > 64;
242
243                if (manyColumns) {
244                        // generate each start point and put a slice in
245                        IndexIterator iter = tdata.getSliceIterator(null, null, shape);
246                        SliceIterator siter = (SliceIterator) tdata.getSliceIterator(null, shape, null);
247                        final int[] pos = iter.getPos();
248                        while (iter.hasNext()) {
249                                siter.setStart(pos);
250                                tdata.setSlice(a, siter);
251                        }
252
253                } else {
254                        // for each value, set slice given by repeats
255                        final int[] skip = new int[rank];
256                        for (int i = 0; i < rank; i++) {
257                                if (reps[i] == 1) {
258                                        skip[i] = newShape[i];
259                                } else {
260                                        skip[i] = shape[i];
261                                }
262                        }
263
264                        Dataset aa = convertToDataset(a);
265                        IndexIterator ita = aa.getIterator(true);
266                        final int[] pos = ita.getPos();
267
268                        final int[] sstart = new int[rank];
269                        final int extra = rank - pos.length;
270                        for (int i = 0; i < extra; i++) {
271                                sstart[i] = 0;
272                        }
273                        SliceIterator siter = (SliceIterator) tdata.getSliceIterator(sstart, null, skip);
274                        while (ita.hasNext()) {
275                                for (int i = 0; i < pos.length; i++) {
276                                        sstart[i + extra] = pos[i];
277                                }
278                                siter.setStart(sstart);
279                                tdata.setSlice(aa.getObjectAbs(ita.index), siter);
280                        }
281                }
282
283                return tdata;
284        }
285
286        /**
287         * Permute copy of dataset's axes so that given order is old order:
288         * <pre>{@literal 
289         *  axisPerm = (p(0), p(1),...) => newdata(n(0), n(1),...) = olddata(o(0), o(1), ...)
290         *  such that n(i) = o(p(i)) for all i
291         * }</pre>
292         * I.e. for a 3D dataset (1,0,2) implies the new dataset has its 1st dimension
293         * running along the old dataset's 2nd dimension and the new 2nd is the old 1st.
294         * The 3rd dimension is left unchanged.
295         *
296         * @param a dataset
297         * @param axes if null or zero length then axes order reversed
298         * @return remapped copy of data
299         */
300        public static Dataset transpose(final IDataset a, int... axes) {
301                return convertToDataset(a).transpose(axes);
302        }
303
304        /**
305         * Swap two axes in dataset
306         * @param a dataset
307         * @param axis1 first axis
308         * @param axis2 second axis
309         * @return swapped dataset
310         */
311        public static Dataset swapAxes(final IDataset a, int axis1, int axis2) {
312                return convertToDataset(a).swapAxes(axis1, axis2);
313        }
314
315        /**
316         * @param <T> dataset class
317         * @param a dataset
318         * @return sorted flattened copy of dataset
319         */
320        public static <T extends Dataset> T sort(final T a) {
321                return sort(a, (Integer) null);
322        }
323
324        /**
325         * @param <T> dataset class
326         * @param a dataset
327         * @param axis to sort along, if null then dataset is first flattened
328         * @return dataset sorted along axis
329         */
330        @SuppressWarnings("unchecked")
331        public static <T extends Dataset> T sort(final T a, final Integer axis) {
332                T s = (T) a.clone();
333                return (T) s.sort(axis);
334        }
335
336        /**
337         * Sort in place given dataset and reorder ancillary datasets too
338         * @param a dataset to be sorted
339         * @param b ancillary datasets
340         */
341        public static void sort(Dataset a, Dataset... b) {
342                if (!InterfaceUtils.isNumerical(a.getClass())) {
343                        throw new UnsupportedOperationException("Sorting non-numerical datasets not supported yet");
344                }
345
346                // gather all datasets as double dataset copies
347                DoubleDataset s = copy(DoubleDataset.class, a);
348                int l = b == null ? 0 : b.length;
349                DoubleDataset[] t = new DoubleDataset[l];
350                int n = 0;
351                for (int i = 0; i < l; i++) {
352                        if (b[i] != null) {
353                                if (!InterfaceUtils.isNumerical(b[i].getClass())) {
354                                        throw new UnsupportedOperationException("Sorting non-numerical datasets not supported yet");
355                                }
356                                t[i] = copy(DoubleDataset.class, b[i]);
357                                n++;
358                        }
359                }
360
361                double[][] y = new double[n][];
362                for (int i = 0, j = 0; i < l; i++) {
363                        if (t[i] != null) {
364                                y[j++] = t[i].getData();
365                        }
366                }
367
368                MathArrays.sortInPlace(s.getData(), y);
369
370                a.setSlice(s);
371                for (int i = 0; i < l; i++) {
372                        if (b[i] != null) {
373                                b[i].setSlice(t[i]);
374                        }
375                }
376        }
377
378        /**
379         * Indirectly sort along given axis
380         * @param a dataset whose indexes will be sorted
381         * @param axis to sort along, if null then dataset is first flattened
382         * @return indexes
383         * @since 2.1
384         */
385        public static IntegerDataset indexSort(Dataset a, Integer axis) {
386                if (axis == null) {
387                        int size = a.getSize();
388                        Integer[] index = new Integer[size];
389                        for (int i = 0; i < size; i++) {
390                                index[i] = i;
391                        }
392                        final Dataset f = a.flatten(); // is this correct for views??? Check with NumPy
393                        Comparator<Integer> cmp = new Comparator<Integer>() {
394
395                                @Override
396                                public int compare(Integer o1, Integer o2) {
397
398                                        return Double.compare(f.getElementDoubleAbs(o1), f.getElementDoubleAbs(o2));
399                                }
400                        };
401                        Arrays.sort(index, cmp);
402                        return DatasetFactory.createFromObject(IntegerDataset.class, index);
403                }
404
405                axis = a.checkAxis(axis);
406                final int[] shape = a.getShapeRef();
407                IntegerDataset id = DatasetFactory.zeros(IntegerDataset.class, shape);
408                int size = shape[axis];
409                Integer[] index = new Integer[size];
410
411                int[] dShape = new int[shape.length];
412                Arrays.fill(dShape, 1);
413                dShape[axis] = size;
414                final DoubleDataset dd = DatasetFactory.zeros(DoubleDataset.class, dShape);
415                final Comparator<Integer> cmp = new Comparator<Integer>() {
416                        @Override
417                        public int compare(Integer o1, Integer o2) {
418
419                                return Double.compare(dd.getElementDoubleAbs(o1), dd.getElementDoubleAbs(o2));
420                        }
421                };
422
423                SliceND ds = new SliceND(dShape);
424                SliceNDIterator it = new SliceNDIterator(new SliceND(shape), axis);
425                int[] pos = it.getPos();
426                int[] ipos = pos.clone();
427                while (it.hasNext()) {
428                        dd.setSlice(a.getSliceView(it.getCurrentSlice()), ds);
429                        for (int i = 0; i < size; i++) {
430                                index[i] = i;
431                        }
432                        Arrays.sort(index, cmp);
433
434                        System.arraycopy(pos, 0, ipos, 0, pos.length);
435                        for (int i = 0; i < size; i++) {
436                                ipos[axis] = i;
437                                id.set(index[i], ipos);
438                        }
439                }
440
441                return id;
442        }
443
444        /**
445         * Concatenate the set of datasets along given axis
446         * @param as datasets
447         * @param axis to concatenate along
448         * @return concatenated dataset
449         */
450        public static Dataset concatenate(final IDataset[] as, int axis) {
451                if (as == null || as.length == 0) {
452                        utilsLogger.error("No datasets given");
453                        throw new IllegalArgumentException("No datasets given");
454                }
455                IDataset a = as[0];
456                if (as.length == 1) {
457                        return convertToDataset(a.clone());
458                }
459
460                int[] ashape = a.getShape();
461                axis = ShapeUtils.checkAxis(ashape.length, axis);
462                Class<? extends Dataset> ac = InterfaceUtils.getInterface(a);
463                int anum = as.length;
464                int isize = a.getElementsPerItem();
465
466                int i = 1;
467                for (; i < anum; i++) {
468                        if (!ac.equals(InterfaceUtils.getInterface(as[i]))) {
469                                utilsLogger.error("Datasets are not of same type");
470                                break;
471                        }
472                        if (!ShapeUtils.areShapesCompatible(ashape, as[i].getShape(), axis)) {
473                                utilsLogger.error("Datasets' shapes are not equal");
474                                break;
475                        }
476                        final int is = as[i].getElementsPerItem();
477                        if (isize < is)
478                                isize = is;
479                }
480                if (i < anum) {
481                        utilsLogger.error("Dataset are not compatible");
482                        throw new IllegalArgumentException("Datasets are not compatible");
483                }
484
485                for (i = 1; i < anum; i++) {
486                        ashape[axis] += as[i].getShape()[axis];
487                }
488
489                Dataset result = DatasetFactory.zeros(isize, ac, ashape);
490
491                int[] start = new int[ashape.length];
492                int[] stop = ashape;
493                stop[axis] = 0;
494                for (i = 0; i < anum; i++) {
495                        IDataset b = as[i];
496                        int[] bshape = b.getShape();
497                        stop[axis] += bshape[axis];
498                        result.setSlice(b, start, stop, null);
499                        start[axis] += bshape[axis];
500                }
501
502                return result;
503        }
504
505        /**
506         * Split a dataset into equal sections along given axis
507         * @param a dataset
508         * @param sections number of sections
509         * @param axis to split along
510         * @param checkEqual makes sure the division is into equal parts
511         * @return list of split datasets
512         */
513        public static List<Dataset> split(final Dataset a, int sections, int axis, final boolean checkEqual) {
514                int[] ashape = a.getShapeRef();
515                axis = a.checkAxis(axis);
516                int imax = ashape[axis];
517                if (checkEqual && (imax%sections) != 0) {
518                        utilsLogger.error("Number of sections does not divide axis into equal parts");
519                        throw new IllegalArgumentException("Number of sections does not divide axis into equal parts");
520                }
521                int n = (imax + sections - 1) / sections;
522                int[] indices = new int[sections-1];
523                for (int i = 1; i < sections; i++)
524                        indices[i-1] = n*i;
525                return split(a, indices, axis);
526        }
527
528        /**
529         * Split a dataset into parts along given axis
530         * @param a dataset
531         * @param indices where to split
532         * @param axis to split along
533         * @return list of split datasets
534         */
535        public static List<Dataset> split(final Dataset a, int[] indices, int axis) {
536                final int[] ashape = a.getShapeRef();
537                axis = a.checkAxis(axis);
538                final int rank = ashape.length;
539                final int imax = ashape[axis];
540
541                final List<Dataset> result = new ArrayList<Dataset>();
542
543                final int[] nshape = ashape.clone();
544                final int is = a.getElementsPerItem();
545
546                int oind = 0;
547                final int[] start = new int[rank];
548                final int[] stop = new int[rank];
549                final int[] step = new int[rank];
550                for (int i = 0; i < rank; i++) {
551                        start[i] = 0;
552                        stop[i] = ashape[i];
553                        step[i] = 1;
554                }
555                for (int ind : indices) {
556                        if (ind > imax) {
557                                result.add(DatasetFactory.zeros(is, a.getClass(), 0));
558                        } else {
559                                nshape[axis] = ind - oind;
560                                start[axis] = oind;
561                                stop[axis] = ind;
562                                Dataset n = DatasetFactory.zeros(is, a.getClass(), nshape);
563                                IndexIterator iter = a.getSliceIterator(start, stop, step);
564
565                                a.fillDataset(n, iter);
566                                result.add(n);
567                                oind = ind;
568                        }
569                }
570
571                if (imax > oind) {
572                        nshape[axis] = imax - oind;
573                        start[axis] = oind;
574                        stop[axis] = imax;
575                        Dataset n = DatasetFactory.zeros(is, a.getClass(), nshape);
576                        IndexIterator iter = a.getSliceIterator(start, stop, step);
577
578                        a.fillDataset(n, iter);
579                        result.add(n);
580                }
581
582                return result;
583        }
584
585        /**
586         * Constructs a dataset which has its elements along an axis replicated from
587         * the original dataset by the number of times given in the repeats array.
588         *
589         * By default, axis=-1 implies using a flattened version of the input dataset
590         *
591         * @param <T> dataset class
592         * @param a dataset
593         * @param repeats number of repetitions
594         * @param axis to repeat
595         * @return dataset
596         */
597        public static <T extends Dataset> T repeat(T a, int[] repeats, int axis) {
598                Serializable buf = a.getBuffer();
599                int[] shape = a.getShape();
600                int rank = shape.length;
601
602                if (axis >= rank) {
603                        utilsLogger.warn("Axis value is out of bounds");
604                        throw new IllegalArgumentException("Axis value is out of bounds");
605                }
606
607                int alen;
608                if (axis < 0) {
609                        alen = a.getSize();
610                        axis = 0;
611                        rank = 1;
612                        shape[0] = alen;
613                } else {
614                        alen = shape[axis];
615                }
616                int rlen = repeats.length;
617                if (rlen != 1 && rlen != alen) {
618                        utilsLogger.warn("Repeats array should have length of 1 or match chosen axis");
619                        throw new IllegalArgumentException("Repeats array should have length of 1 or match chosen axis");
620                }
621
622                for (int i = 0; i < rlen; i++) {
623                        if (repeats[i] < 0) {
624                                utilsLogger.warn("Negative repeat value is not allowed");
625                                throw new IllegalArgumentException("Negative repeat value is not allowed");
626                        }
627                }
628
629                int[] newShape = new int[rank];
630                for (int i = 0; i < rank; i ++)
631                        newShape[i] = shape[i];
632
633                // do single repeat separately
634                if (repeats.length == 1) {
635                        newShape[axis] *= repeats[0];
636                } else {
637                        int nlen = 0;
638                        for (int i = 0; i < alen; i++) {
639                                nlen += repeats[i];
640                        }
641                        newShape[axis] = nlen;
642                }
643
644                T rdata = DatasetFactory.zeros(a, newShape);
645                Serializable nbuf = rdata.getBuffer();
646
647                int csize = a.getElementsPerItem(); // chunk size
648                for (int i = axis+1; i < rank; i++) {
649                        csize *= newShape[i];
650                }
651                int nout = 1;
652                for (int i = 0; i < axis; i++) {
653                        nout *= newShape[i];
654                }
655
656                int oi = 0;
657                int ni = 0;
658                if (rlen == 1) { // do single repeat separately
659                        for (int i = 0; i < nout; i++) {
660                                for (int j = 0; j < shape[axis]; j++) {
661                                        for (int k = 0; k < repeats[0]; k++) {
662                                                System.arraycopy(buf, oi, nbuf, ni, csize);
663                                                ni += csize;
664                                        }
665                                        oi += csize;
666                                }
667                        }
668                } else {
669                        for (int i = 0; i < nout; i++) {
670                                for (int j = 0; j < shape[axis]; j++) {
671                                        for (int k = 0; k < repeats[j]; k++) {
672                                                System.arraycopy(buf, oi, nbuf, ni, csize);
673                                                ni += csize;
674                                        }
675                                        oi += csize;
676                                }
677                        }
678                }
679
680                return rdata;
681        }
682
683        /**
684         * Resize a dataset
685         * @param <T> dataset class
686         * @param a dataset
687         * @param shape output shape
688         * @return new dataset with new shape and items that are truncated or repeated, as necessary
689         */
690        public static <T extends Dataset> T resize(final T a, final int... shape) {
691                int size = a.getSize();
692                T rdata = DatasetFactory.zeros(a, shape);
693                IndexIterator it = rdata.getIterator();
694                while (it.hasNext()) {
695                        rdata.setObjectAbs(it.index, a.getObjectAbs(it.index % size));
696                }
697
698                return rdata;
699        }
700
701        /**
702         * Copy and cast a dataset
703         *
704         * @param d
705         *            The dataset to be copied
706         * @param dtype dataset type
707         * @return copied dataset of given type
708         * @deprecated Please use the class-based methods in DatasetUtils,
709         *             such as {@link #copy(Class, IDataset)}
710         */
711        @Deprecated
712        public static Dataset copy(final IDataset d, final int dtype) {
713                return copy(DTypeUtils.getInterface(dtype), d);
714        }
715
716        /**
717         * Cast a dataset
718         *
719         * @param <T> dataset sub-interface
720         * @param clazz dataset sub-interface
721         * @param d
722         *            The dataset to be copied
723         * @return dataset of given class (or same dataset if already of the right class)
724         */
725        @SuppressWarnings("unchecked")
726        public static <T extends Dataset> T copy(Class<T> clazz, final IDataset d) {
727                Dataset a = convertToDataset(d);
728                Dataset c = null;
729                try {
730                        // copy across the data
731                        if (BooleanDataset.class.isAssignableFrom(clazz)) {
732                                c = new BooleanDataset(a);
733                        } else if (ByteDataset.class.isAssignableFrom(clazz)) {
734                                c = new ByteDataset(a);
735                        } else if (ShortDataset.class.isAssignableFrom(clazz)) {
736                                c = new ShortDataset(a);
737                        } else if (IntegerDataset.class.isAssignableFrom(clazz)) {
738                                c = new IntegerDataset(a);
739                        } else if (LongDataset.class.isAssignableFrom(clazz)) {
740                                c = new LongDataset(a);
741                        } else if (RGBByteDataset.class.isAssignableFrom(clazz)) {
742                                if (a instanceof CompoundDataset) {
743                                        c = new RGBByteDataset((CompoundDataset) a);
744                                } else {
745                                        c = new RGBByteDataset(a);
746                                }
747                        } else if (CompoundByteDataset.class.isAssignableFrom(clazz)) {
748                                if (a instanceof CompoundByteDataset) {
749                                        c = new CompoundByteDataset((CompoundDataset) a);
750                                } else {
751                                        c = new CompoundByteDataset(a);
752                                }
753                        } else if (RGBDataset.class.isAssignableFrom(clazz)) {
754                                if (a instanceof CompoundDataset) {
755                                        c = new RGBDataset((CompoundDataset) a);
756                                } else {
757                                        c = new RGBDataset(a);
758                                }
759                        } else if (CompoundShortDataset.class.isAssignableFrom(clazz)) {
760                                if (a instanceof CompoundDataset) {
761                                        c = new CompoundShortDataset((CompoundDataset) a);
762                                } else {
763                                        c = new CompoundShortDataset(a);
764                                }
765                        } else if (CompoundIntegerDataset.class.isAssignableFrom(clazz)) {
766                                if (a instanceof CompoundDataset) {
767                                        c = new CompoundIntegerDataset((CompoundDataset) a);
768                                } else {
769                                        c = new CompoundIntegerDataset(a);
770                                }
771                        } else if (CompoundLongDataset.class.isAssignableFrom(clazz)) {
772                                if (a instanceof CompoundDataset) {
773                                        c = new CompoundLongDataset((CompoundDataset) a);
774                                } else {
775                                        c = new CompoundLongDataset(a);
776                                }
777                        } else if (FloatDataset.class.isAssignableFrom(clazz)) {
778                                c = new FloatDataset(a);
779                        } else if (DoubleDataset.class.isAssignableFrom(clazz)) {
780                                c = new DoubleDataset(a);
781                        } else if (ComplexFloatDataset.class.isAssignableFrom(clazz)) {
782                                c = new ComplexFloatDataset(a);
783                        } else if (ComplexDoubleDataset.class.isAssignableFrom(clazz)) {
784                                c = new ComplexDoubleDataset(a);
785                        } else if (CompoundFloatDataset.class.isAssignableFrom(clazz)) {
786                                if (a instanceof CompoundDataset) {
787                                        c = new CompoundFloatDataset((CompoundDataset) a);
788                                } else {
789                                        c = new CompoundFloatDataset(a);
790                                }
791                        } else if (CompoundDoubleDataset.class.isAssignableFrom(clazz)) {
792                                if (a instanceof CompoundDataset) {
793                                        c = new CompoundDoubleDataset((CompoundDataset) a);
794                                } else {
795                                        c = new CompoundDoubleDataset(a);
796                                }
797                        } else if (DateDataset.class.isAssignableFrom(clazz)) {
798                                c = DateDatasetImpl.createFromObject(a);
799                        } else if (StringDataset.class.isAssignableFrom(clazz)) {
800                                c = new StringDataset(a);
801                        } else if (ObjectDataset.class.isAssignableFrom(clazz)) {
802                                c = new ObjectDataset(a);
803                        } else {
804                                utilsLogger.error("Dataset of unknown type!");
805                        }
806                } catch (OutOfMemoryError e) {
807                        utilsLogger.error("Not enough memory available to create dataset");
808                        throw new OutOfMemoryError("Not enough memory available to create dataset");
809                }
810
811                return (T) c;
812        }
813
814        /**
815         * Cast a dataset
816         *
817         * @param d
818         *            The dataset to be cast
819         * @param dtype dataset type
820         * @return dataset of given type (or same dataset if already of the right type)
821         * @deprecated Please use the class-based methods in DatasetUtils,
822         *             such as {@link #cast(Class, IDataset)}
823         */
824        @Deprecated
825        public static Dataset cast(final IDataset d, final int dtype) {
826                return cast(DTypeUtils.getInterface(dtype), d);
827        }
828
829        /**
830         * Cast a dataset
831         * @param <T> dataset sub-interface
832         * @param clazz dataset sub-interface
833         * @param d
834         *            The dataset to be cast
835         * @return dataset of given class
836         */
837        @SuppressWarnings("unchecked")
838        public static <T extends Dataset> T cast(Class<T> clazz, final IDataset d) {
839                Dataset a = convertToDataset(d);
840
841                if (a.getClass().equals(clazz)) {
842                        return (T) a;
843                }
844
845                if (a instanceof CompoundDataset) {
846                        if (RGBByteDataset.class.isAssignableFrom(clazz)) {
847                                return (T) RGBByteDataset.createFromCompoundDataset((CompoundDataset) a);
848                        }
849                        if (RGBDataset.class.isAssignableFrom(clazz)) {
850                                return (T) RGBDataset.createFromCompoundDataset((CompoundDataset) a);
851                        }
852                }
853
854                return copy(clazz, d);
855        }
856
857        /**
858         * Cast a dataset
859         *
860         * @param d
861         *            The dataset to be cast
862         * @param repeat repeat elements over item
863         * @param dtype dataset type
864         * @param isize item size
865         * @return dataset of given type
866         * @deprecated Please use the class-based methods in DatasetUtils,
867         *             such as {@link #cast(Class, IDataset)}
868         */
869        @Deprecated
870        public static Dataset cast(final IDataset d, final boolean repeat, final int dtype, final int isize) {
871                return cast(isize, DTypeUtils.getInterface(dtype), d, repeat);
872        }
873
874        /**
875         * Cast a dataset
876         *
877         * @param <T> dataset sub-interface
878         * @param isize item size
879         * @param clazz dataset sub-interface
880         * @param d
881         *            The dataset to be cast.
882         * @param repeat repeat elements over item
883         * @return dataset of given class
884         * @since 2.3
885         */
886        @SuppressWarnings("unchecked")
887        public static <T extends Dataset> T cast(final int isize, Class<T> clazz, final IDataset d, final boolean repeat) {
888                Dataset a = convertToDataset(d);
889
890                if (a.getClass().equals(clazz) && a.getElementsPerItem() == isize) {
891                        return (T) a;
892                }
893                if (isize <= 0) {
894                        utilsLogger.error("Item size is invalid (>0)");
895                        throw new IllegalArgumentException("Item size is invalid (>0)");
896                }
897                if (isize > 1 && !InterfaceUtils.isComplex(clazz) && InterfaceUtils.isElemental(clazz)) {
898                        utilsLogger.error("Item size is inconsistent with dataset type");
899                        throw new IllegalArgumentException("Item size is inconsistent with dataset type");
900                }
901
902                Dataset c = null;
903
904                try {
905                        // copy across the data
906                        if (BooleanDataset.class.isAssignableFrom(clazz)) {
907                                c = new BooleanDataset(a);
908                        } else if (ByteDataset.class.isAssignableFrom(clazz)) {
909                                c = new ByteDataset(a);
910                        } else if (ShortDataset.class.isAssignableFrom(clazz)) {
911                                c = new ShortDataset(a);
912                        } else if (IntegerDataset.class.isAssignableFrom(clazz)) {
913                                c = new IntegerDataset(a);
914                        } else if (LongDataset.class.isAssignableFrom(clazz)) {
915                                c = new LongDataset(a);
916                        } else if (RGBByteDataset.class.isAssignableFrom(clazz)) {
917                                if (a instanceof CompoundDataset) {
918                                        c = RGBByteDataset.createFromCompoundDataset((CompoundDataset) a);
919                                } else {
920                                        c = new RGBByteDataset(a);
921                                }
922                        } else if (CompoundByteDataset.class.isAssignableFrom(clazz)) {
923                                c = new CompoundByteDataset(isize, repeat, a);
924                        } else if (RGBDataset.class.isAssignableFrom(clazz)) {
925                                if (a instanceof CompoundDataset) {
926                                        c = RGBDataset.createFromCompoundDataset((CompoundDataset) a);
927                                } else {
928                                        c = new RGBDataset(a);
929                                }
930                        } else if (CompoundShortDataset.class.isAssignableFrom(clazz)) {
931                                c = new CompoundShortDataset(isize, repeat, a);
932                        } else if (CompoundIntegerDataset.class.isAssignableFrom(clazz)) {
933                                c = new CompoundIntegerDataset(isize, repeat, a);
934                        } else if (CompoundLongDataset.class.isAssignableFrom(clazz)) {
935                                c = new CompoundLongDataset(isize, repeat, a);
936                        } else if (FloatDataset.class.isAssignableFrom(clazz)) {
937                                c = new FloatDataset(a);
938                        } else if (DoubleDataset.class.isAssignableFrom(clazz)) {
939                                c = new DoubleDataset(a);
940                        } else if (ComplexFloatDataset.class.isAssignableFrom(clazz)) {
941                                c = new ComplexFloatDataset(a);
942                        } else if (ComplexDoubleDataset.class.isAssignableFrom(clazz)) {
943                                c = new ComplexDoubleDataset(a);
944                        } else if (CompoundFloatDataset.class.isAssignableFrom(clazz)) {
945                                c = new CompoundFloatDataset(isize, repeat, a);
946                        } else if (CompoundDoubleDataset.class.isAssignableFrom(clazz)) {
947                                c = new CompoundDoubleDataset(isize, repeat, a);
948                        } else if (DateDataset.class.isAssignableFrom(clazz)) {
949                                c = DateDatasetImpl.createFromObject(a);
950                        } else if (StringDataset.class.isAssignableFrom(clazz)) {
951                                c = new StringDataset(a);
952                        } else if (ObjectDataset.class.isAssignableFrom(clazz)) {
953                                c = new ObjectDataset(a);
954                        } else {
955                                utilsLogger.error("Dataset of unknown type!");
956                        }
957                } catch (OutOfMemoryError e) {
958                        utilsLogger.error("Not enough memory available to create dataset");
959                        throw new OutOfMemoryError("Not enough memory available to create dataset");
960                }
961
962                return (T) c;
963        }
964
965        /**
966         * Cast array of datasets to a compound dataset
967         *
968         * @param <T> compound dataset sub-interface
969         * @param clazz compound dataset sub-interface
970         * @param a
971         *            The datasets to be cast.
972         * @return dataset of given class
973         * @since 2.3
974         */
975        public static  <T extends CompoundDataset> T compoundCast(Class<T> clazz, final Dataset... a) {
976                return createCompoundDataset(clazz, a);
977        }
978
979
980        /**
981         * Cast array of datasets to a compound dataset
982         *
983         * @param clazz dataset class
984         * @param a
985         *            The datasets to be cast.
986         * @return compound dataset of given class
987         * @since 2.3
988         */
989        public static  CompoundDataset cast(Class<? extends Dataset> clazz, final Dataset... a) {
990                return compoundCast(InterfaceUtils.getCompoundInterface(clazz), a);
991        }
992
993        /**
994         * Cast array of datasets to a compound dataset
995         *
996         * @param a
997         *            The datasets to be cast.
998         * @param dtype dataset type
999         * @return compound dataset of given type
1000         * @deprecated Please use the class-based methods in DatasetUtils,
1001         *             such as {@link #cast(Class, Dataset...)}
1002         */
1003        @Deprecated
1004        public static CompoundDataset cast(final Dataset[] a, final int dtype) {
1005                return cast(DTypeUtils.getInterface(dtype), a);
1006        }
1007
1008        /**
1009         * Make a dataset unsigned by promoting it to a wider dataset type and unwrapping the signs
1010         * of its contents
1011         * @param a dataset
1012         * @return unsigned dataset or original if it is not an integer dataset
1013         */
1014        public static Dataset makeUnsigned(IDataset a) {
1015                return makeUnsigned(a, false);
1016        }
1017
1018        /**
1019         * Make a dataset unsigned by promoting it to a wider dataset type and unwrapping the signs
1020         * of its contents
1021         * @param a dataset
1022         * @param check if true, then check for negative values
1023         * @return unsigned dataset or original if it is not an integer dataset or it has been check for negative numbers
1024         * @since 2.1
1025         */
1026        public static Dataset makeUnsigned(IDataset a, boolean check) {
1027                Dataset d = convertToDataset(a);
1028
1029                if (d.hasFloatingPointElements()) {
1030                        return d;
1031                }
1032                if (check && d.min(true).longValue() >= 0) {
1033                        return d;
1034                }
1035
1036                if (d instanceof ByteDataset) {
1037                        d = new ShortDataset(d);
1038                        unwrapUnsigned(d, 8);
1039                } else if (d instanceof ShortDataset) {
1040                        d = new IntegerDataset(d);
1041                        unwrapUnsigned(d, 16);
1042                } else if (d instanceof IntegerDataset) {
1043                        d = new LongDataset(d);
1044                        unwrapUnsigned(d, 32);
1045                } else if (d instanceof CompoundByteDataset) {
1046                        d = new CompoundShortDataset(d);
1047                        unwrapUnsigned(d, 8);
1048                } else if (d instanceof CompoundShortDataset) {
1049                        d = new CompoundIntegerDataset(d);
1050                        unwrapUnsigned(d, 16);
1051                } else if (d instanceof CompoundIntegerDataset) {
1052                        d = new CompoundLongDataset(d);
1053                        unwrapUnsigned(d, 32);
1054                }
1055                return d;
1056        }
1057
1058        /**
1059         * Unwrap dataset elements so that all elements are unsigned
1060         * @param a dataset
1061         * @param bitWidth width of original primitive in bits
1062         */
1063        public static void unwrapUnsigned(Dataset a, final int bitWidth) {
1064                final double dv = 1L << bitWidth;
1065                final int isize = a.getElementsPerItem();
1066                IndexIterator it = a.getIterator();
1067
1068                if (a instanceof ShortDataset) {
1069                        ShortDataset sds = (ShortDataset) a;
1070                        final short soffset = (short) dv;
1071                        while (it.hasNext()) {
1072                                final short x = sds.getAbs(it.index);
1073                                if (x < 0) {
1074                                        sds.setAbs(it.index, (short) (x + soffset));
1075                                }
1076                        }
1077                } else if (a instanceof IntegerDataset) {
1078                        IntegerDataset ids = (IntegerDataset) a;
1079                        final int ioffset = (int) dv;
1080                        while (it.hasNext()) {
1081                                final int x = ids.getAbs(it.index);
1082                                if (x < 0) {
1083                                        ids.setAbs(it.index, x + ioffset);
1084                                }
1085                        }
1086                } else if (a instanceof LongDataset) {
1087                        LongDataset lds = (LongDataset) a;
1088                        final long loffset = (long) dv;
1089                        while (it.hasNext()) {
1090                                final long x = lds.getAbs(it.index);
1091                                if (x < 0) {
1092                                        lds.setAbs(it.index, x + loffset);
1093                                }
1094                        }
1095                } else if (a instanceof CompoundShortDataset) {
1096                        CompoundShortDataset csds = (CompoundShortDataset) a;
1097                        final short csoffset = (short) dv;
1098                        final short[] csa = new short[isize];
1099                        while (it.hasNext()) {
1100                                csds.getAbs(it.index, csa);
1101                                boolean dirty = false;
1102                                for (int i = 0; i < isize; i++) {
1103                                        short x = csa[i];
1104                                        if (x < 0) {
1105                                                csa[i] = (short) (x + csoffset);
1106                                                dirty = true;
1107                                        }
1108                                }
1109                                if (dirty) {
1110                                        csds.setAbs(it.index, csa);
1111                                }
1112                        }
1113                } else if (a instanceof CompoundIntegerDataset) {
1114                        CompoundIntegerDataset cids = (CompoundIntegerDataset) a;
1115                        final int cioffset = (int) dv;
1116                        final int[] cia = new int[isize];
1117                        while (it.hasNext()) {
1118                                cids.getAbs(it.index, cia);
1119                                boolean dirty = false;
1120                                for (int i = 0; i < isize; i++) {
1121                                        int x = cia[i];
1122                                        if (x < 0) {
1123                                                cia[i] = x + cioffset;
1124                                                dirty = true;
1125                                        }
1126                                }
1127                                if (dirty) {
1128                                        cids.setAbs(it.index, cia);
1129                                }
1130                        }
1131                } else if (a instanceof CompoundLongDataset) {
1132                        CompoundLongDataset clds = (CompoundLongDataset) a;
1133                        final long cloffset = (long) dv;
1134                        final long[] cla = new long[isize];
1135                        while (it.hasNext()) {
1136                                clds.getAbs(it.index, cla);
1137                                boolean dirty = false;
1138                                for (int i = 0; i < isize; i++) {
1139                                        long x = cla[i];
1140                                        if (x < 0) {
1141                                                cla[i] = x + cloffset;
1142                                                dirty = true;
1143                                        }
1144                                }
1145                                if (dirty) {
1146                                        clds.setAbs(it.index, cla);
1147                                }
1148                        }
1149                }
1150        }
1151
1152        /**
1153         * @param <T> dataset sub-interface
1154         * @param clazz dataset sub-interface
1155         * @param rows number of rows
1156         * @param cols number of columns
1157         * @param offset row offset
1158         * @return a new 2d dataset of given shape and class, filled with ones on the (offset) diagonal
1159         * @since 2.3
1160         */
1161        public static <T extends Dataset> T eye(final Class<T> clazz, final int rows, final int cols, final int offset) {
1162                int[] shape = new int[] {rows, cols};
1163                T a = DatasetFactory.zeros(clazz, shape);
1164
1165                int[] pos = new int[] {0, offset};
1166                while (pos[1] < 0) {
1167                        pos[0]++;
1168                        pos[1]++;
1169                }
1170                while (pos[0] < rows && pos[1] < cols) {
1171                        a.set(1, pos);
1172                        pos[0]++;
1173                        pos[1]++;
1174                }
1175
1176                return a;
1177        }
1178
1179        /**
1180         * @param rows number of rows
1181         * @param cols number of columns
1182         * @param offset row offset
1183         * @param dtype dataset type
1184         * @return a new 2d dataset of given shape and type, filled with ones on the (offset) diagonal
1185         * @deprecated Please use the class-based methods in DatasetUtils,
1186         *             such as {@link #eye(Class, int, int, int)}
1187         */
1188        @Deprecated
1189        public static Dataset eye(final int rows, final int cols, final int offset, final int dtype) {
1190                return eye(DTypeUtils.getInterface(dtype), rows, cols, offset);
1191        }
1192
1193        /**
1194         * Create a (off-)diagonal matrix from items in dataset
1195         *
1196         * @param <T> dataset class
1197         * @param a dataset
1198         * @param offset distance right of diagonal
1199         * @return diagonal matrix
1200         */
1201        public static <T extends Dataset> T diag(final T a, final int offset) {
1202                final int rank = a.getRank();
1203
1204                if (rank == 0 || rank > 2) {
1205                        utilsLogger.error("Rank of dataset should be one or two");
1206                        throw new IllegalArgumentException("Rank of dataset should be one or two");
1207                }
1208
1209                T result;
1210                final int[] shape = a.getShapeRef();
1211                if (rank == 1) {
1212                        int side = shape[0] + Math.abs(offset);
1213                        int[] pos = new int[] {side, side};
1214                        result = DatasetFactory.zeros(a, pos);
1215                        if (offset >= 0) {
1216                                pos[0] = 0;
1217                                pos[1] = offset;
1218                        } else {
1219                                pos[0] = -offset;
1220                                pos[1] = 0;
1221                        }
1222                        int i = 0;
1223                        while (pos[0] < side && pos[1] < side) {
1224                                result.set(a.getObject(i++), pos);
1225                                pos[0]++;
1226                                pos[1]++;
1227                        }
1228                } else {
1229                        int side = offset >= 0 ? Math.min(shape[0], shape[1]-offset) : Math.min(shape[0]+offset, shape[1]);
1230                        if (side < 0)
1231                                side = 0;
1232                        result = DatasetFactory.zeros(a, side, side);
1233
1234                        if (side > 0) {
1235                                int[] pos = offset >= 0 ? new int[] { 0, offset } : new int[] { -offset, 0 };
1236                                int i = 0;
1237                                while (pos[0] < shape[0] && pos[1] < shape[1]) {
1238                                        result.set(a.getObject(pos), i++);
1239                                        pos[0]++;
1240                                        pos[1]++;
1241                                }
1242                        }
1243                }
1244
1245                return (T) result;
1246        }
1247
1248        /**
1249         * Slice (or fully load), if necessary, a lazy dataset, otherwise take a slice view and
1250         * convert to our dataset implementation. If a slice is necessary, this may cause resource
1251         * problems when used on large datasets and throw runtime exceptions
1252         * @param lazy can be null
1253         * @return Converted dataset or null
1254         * @throws DatasetException when cannot retrieve data
1255         */
1256        public static Dataset sliceAndConvertLazyDataset(ILazyDataset lazy) throws DatasetException {
1257                if (lazy == null)
1258                        return null;
1259
1260                IDataset data = lazy instanceof IDataset ? (IDataset) lazy.getSliceView() : lazy.getSlice();
1261
1262                return convertToDataset(data);
1263        }
1264
1265        /**
1266         * Convert (if necessary) a dataset obeying the interface to our implementation
1267         * @param data can be null
1268         * @return Converted dataset or null
1269         */
1270        public static Dataset convertToDataset(IDataset data) {
1271                if (data == null)
1272                        return null;
1273
1274                if (data instanceof Dataset) {
1275                        return (Dataset) data;
1276                }
1277
1278                final int isize = data.getElementsPerItem();
1279                Class<? extends Dataset> clazz = InterfaceUtils.getInterfaceFromClass(isize, data.getElementClass());
1280                if (isize <= 0) {
1281                        throw new IllegalArgumentException("Datasets with " + isize + " elements per item not supported");
1282                }
1283
1284                final Dataset result = DatasetFactory.zeros(isize, clazz, data.getShape());
1285                result.setName(data.getName());
1286
1287                final IndexIterator it = result.getIterator(true);
1288                final int[] pos = it.getPos();
1289                if (BooleanDataset.class.isAssignableFrom(clazz)) {
1290                        while (it.hasNext()) {
1291                                result.setObjectAbs(it.index, data.getBoolean(pos));
1292                        }
1293                } else if (ByteDataset.class.isAssignableFrom(clazz)) {
1294                        while (it.hasNext()) {
1295                                result.setObjectAbs(it.index, data.getByte(pos));
1296                        }
1297                } else if (ShortDataset.class.isAssignableFrom(clazz)) {
1298                        while (it.hasNext()) {
1299                                result.setObjectAbs(it.index, data.getShort(pos));
1300                        }
1301                } else if (IntegerDataset.class.isAssignableFrom(clazz)) {
1302                        while (it.hasNext()) {
1303                                result.setObjectAbs(it.index, data.getInt(pos));
1304                        }
1305                } else if (LongDataset.class.isAssignableFrom(clazz)) {
1306                        while (it.hasNext()) {
1307                                result.setObjectAbs(it.index, data.getLong(pos));
1308                        }
1309                } else if (FloatDataset.class.isAssignableFrom(clazz)) {
1310                        while (it.hasNext()) {
1311                                result.setObjectAbs(it.index, data.getFloat(pos));
1312                        }
1313                } else if (DoubleDataset.class.isAssignableFrom(clazz)) {
1314                        while (it.hasNext()) {
1315                                result.setObjectAbs(it.index, data.getDouble(pos));
1316                        }
1317                } else {
1318                        while (it.hasNext()) {
1319                                result.setObjectAbs(it.index, data.getObject(pos));
1320                        }
1321                }
1322
1323                result.setErrors(data.getErrors());
1324                return result;
1325        }
1326
1327        /**
1328         * Create a compound dataset from given datasets
1329         * @param datasets inputs
1330         * @return compound dataset or null if none given
1331         */
1332        public static CompoundDataset createCompoundDataset(final Dataset... datasets) {
1333                if (datasets == null || datasets.length == 0)
1334                        return null;
1335
1336                return (CompoundDataset) createCompoundDataset(InterfaceUtils.getCompoundInterface(datasets[0].getClass()), datasets);
1337        }
1338
1339        /**
1340         * Create a compound dataset from copying given datasets
1341         * @param <T> compound dataset sub-interface
1342         * @param clazz compound dataset sub-interface
1343         * @param datasets inputs
1344         * @return compound dataset or null if none given
1345         */
1346        @SuppressWarnings("unchecked")
1347        public static <T extends CompoundDataset> T createCompoundDataset(Class<T> clazz, final Dataset... datasets) {
1348                if (datasets == null || datasets.length == 0)
1349                        return null;
1350
1351                CompoundDataset c = null;
1352                if (RGBByteDataset.class.isAssignableFrom(clazz)) {
1353                        if (datasets.length == 1) {
1354                                c = new RGBByteDataset(datasets[0]);
1355                        } else if (datasets.length == 3) {
1356                                c = new RGBByteDataset(datasets[0], datasets[1], datasets[2]);
1357                        } else {
1358                                throw new IllegalArgumentException("Need one or three datasets for RGB dataset");
1359                        }
1360                } else if (CompoundByteDataset.class.isAssignableFrom(clazz)) {
1361                        c = new CompoundByteDataset(datasets);
1362                } else if (RGBDataset.class.isAssignableFrom(clazz)) {
1363                        if (datasets.length == 1) {
1364                                c = new RGBDataset(datasets[0]);
1365                        } else if (datasets.length == 3) {
1366                                c = new RGBDataset(datasets[0], datasets[1], datasets[2]);
1367                        } else {
1368                                throw new IllegalArgumentException("Need one or three datasets for RGB dataset");
1369                        }
1370                } else if (CompoundShortDataset.class.isAssignableFrom(clazz)) {
1371                        c = new CompoundShortDataset(datasets);
1372                } else if (CompoundIntegerDataset.class.isAssignableFrom(clazz)) {
1373                        c = new CompoundIntegerDataset(datasets);
1374                } else if (CompoundLongDataset.class.isAssignableFrom(clazz)) {
1375                        c = new CompoundLongDataset(datasets);
1376                } else if (ComplexFloatDataset.class.isAssignableFrom(clazz)) {
1377                        if (datasets.length == 1) {
1378                                c = new ComplexFloatDataset(datasets[0]);
1379                        } else if (datasets.length >= 2) {
1380                                c = new ComplexFloatDataset(datasets[0], datasets[1]);
1381                        } else {
1382                                throw new IllegalArgumentException("Need one or more datasets for complex dataset type");
1383                        }
1384                } else if (ComplexDoubleDataset.class.isAssignableFrom(clazz)) {
1385                        if (datasets.length == 1) {
1386                                c = new ComplexDoubleDataset(datasets[0]);
1387                        } else if (datasets.length >= 2) {
1388                                c = new ComplexDoubleDataset(datasets[0], datasets[1]);
1389                        } else {
1390                                throw new IllegalArgumentException("Need one or more datasets for complex dataset type");
1391                        }
1392                } else if (CompoundFloatDataset.class.isAssignableFrom(clazz)) {
1393                        c = new CompoundFloatDataset(datasets);
1394                } else if (DoubleDataset.class.equals(clazz) || CompoundDoubleDataset.class.isAssignableFrom(clazz)) {
1395                        c = new CompoundDoubleDataset(datasets);
1396                } else {
1397                        utilsLogger.error("Dataset of unsupported interface");
1398                }
1399                return (T) c;
1400        }
1401
1402        /**
1403         * Create a compound dataset from given datasets
1404         * @param dtype dataset type
1405         * @param datasets for each element
1406         * @return compound dataset or null if none given
1407         * @deprecated Please use the class-based methods in DatasetUtils,
1408         *             such as {@link #createCompoundDataset(Class, Dataset...)}
1409         */
1410        @Deprecated
1411        public static CompoundDataset createCompoundDataset(final int dtype, final Dataset... datasets) {
1412                return createCompoundDataset(InterfaceUtils.getCompoundInterface(DTypeUtils.getInterface(dtype)), datasets);
1413        }
1414
1415        /**
1416         * Create a compound dataset from given dataset, sharing data
1417         * @param dataset input
1418         * @param itemSize item size
1419         * @return compound dataset
1420         */
1421        public static CompoundDataset createCompoundDataset(final Dataset dataset, final int itemSize) {
1422                int[] shape = dataset.getShapeRef();
1423                int[] nshape = shape;
1424                if (shape != null && itemSize > 1) {
1425                        int size = ShapeUtils.calcSize(shape);
1426                        if (size % itemSize != 0) {
1427                                throw new IllegalArgumentException("Input dataset has number of items that is not a multiple of itemSize");
1428                        }
1429                        int d = shape.length;
1430                        int l = 1;
1431                        while (--d >= 0) {
1432                                l *= shape[d];
1433                                if (l % itemSize == 0) {
1434                                        break;
1435                                }
1436                        }
1437                        assert d >= 0;
1438                        nshape = new int[d + 1];
1439                        for (int i = 0; i < d; i++) {
1440                                nshape[i] = shape[i];
1441                        }
1442                        nshape[d] = l / itemSize;
1443                }
1444
1445                if (dataset instanceof ByteDataset) {
1446                        return new CompoundByteDataset(itemSize, (byte[]) dataset.getBuffer(), nshape);
1447                } else if (dataset instanceof ShortDataset) {
1448                        return new CompoundShortDataset(itemSize, (short[]) dataset.getBuffer(), nshape);
1449                } else if (dataset instanceof IntegerDataset) {
1450                        return new CompoundIntegerDataset(itemSize, (int[]) dataset.getBuffer(), nshape);
1451                } else if (dataset instanceof LongDataset) {
1452                        return new CompoundLongDataset(itemSize, (long[]) dataset.getBuffer(), nshape);
1453                } else if (dataset instanceof FloatDataset) {
1454                        return new CompoundFloatDataset(itemSize, (float[]) dataset.getBuffer(), nshape);
1455                } else if (dataset instanceof DoubleDataset) {
1456                        return new CompoundDoubleDataset(itemSize, (double[]) dataset.getBuffer(), nshape);
1457                } 
1458
1459                utilsLogger.error("Dataset interface not supported for this operation");
1460                throw new UnsupportedOperationException("Dataset interface not supported");
1461        }
1462
1463
1464        /**
1465         * Create a compound dataset by using last axis as elements of an item
1466         * @param a dataset
1467         * @param shareData if true, then share data
1468         * @return compound dataset
1469         */
1470        public static CompoundDataset createCompoundDatasetFromLastAxis(final Dataset a, final boolean shareData) {
1471                if (a instanceof ByteDataset) {
1472                        return CompoundByteDataset.createCompoundDatasetWithLastDimension(a, shareData);
1473                } else if (a instanceof ShortDataset) {
1474                        return CompoundShortDataset.createCompoundDatasetWithLastDimension(a, shareData);
1475                } else if (a instanceof IntegerDataset) {
1476                        return CompoundIntegerDataset.createCompoundDatasetWithLastDimension(a, shareData);
1477                } else if (a instanceof LongDataset) {
1478                        return CompoundLongDataset.createCompoundDatasetWithLastDimension(a, shareData);
1479                } else if (a instanceof FloatDataset) {
1480                        return CompoundFloatDataset.createCompoundDatasetWithLastDimension(a, shareData);
1481                } else if (a instanceof DoubleDataset) {
1482                        return CompoundDoubleDataset.createCompoundDatasetWithLastDimension(a, shareData);
1483                }
1484
1485                utilsLogger.error("Dataset interface not supported for this operation");
1486                throw new UnsupportedOperationException("Dataset interface not supported");
1487        }
1488
1489        /**
1490         * Create a dataset from a compound dataset by using elements of an item as last axis
1491         * <p>
1492         * In the case where the number of elements is one, the last axis is squeezed out.
1493         * @param a dataset
1494         * @param shareData if true, then share data
1495         * @return non-compound dataset
1496         */
1497        public static Dataset createDatasetFromCompoundDataset(final CompoundDataset a, final boolean shareData) {
1498                return a.asNonCompoundDataset(shareData);
1499        }
1500
1501        /**
1502         * Create a copy that has been coerced to an appropriate dataset type
1503         * depending on the input object's class
1504         *
1505         * @param a dataset
1506         * @param obj input object
1507         * @return coerced copy of dataset
1508         */
1509        public static Dataset coerce(Dataset a, Object obj) {
1510                return cast(InterfaceUtils.getBestInterface(a.getClass(), InterfaceUtils.getInterface(obj)), a.clone());
1511        }
1512
1513        /**
1514         * Function that returns a normalised dataset which is bounded between 0 and 1
1515         * @param a dataset
1516         * @return normalised dataset
1517         */
1518        public static Dataset norm(Dataset a) {
1519                double amin = a.min().doubleValue();
1520                double aptp = a.max().doubleValue() - amin;
1521                Dataset temp = Maths.subtract(a, amin);
1522                temp.idivide(aptp);
1523                return temp;
1524        }
1525
1526        /**
1527         * Function that returns a normalised compound dataset which is bounded between 0 and 1. There
1528         * are (at least) two ways to normalise a compound dataset: per element - extrema for each element
1529         * in a compound item is used, i.e. many min/max pairs; over all elements - extrema for all elements
1530         * is used, i.e. one min/max pair.
1531         * @param a dataset
1532         * @param overAllElements if true, then normalise over all elements in each item
1533         * @return normalised dataset
1534         */
1535        public static CompoundDataset norm(CompoundDataset a, boolean overAllElements) {
1536                double[] amin = a.minItem();
1537                double[] amax = a.maxItem();
1538                final int is = a.getElementsPerItem();
1539                Dataset result;
1540
1541                if (overAllElements) {
1542                        Arrays.sort(amin);
1543                        Arrays.sort(amax);
1544                        double aptp = amax[0] - amin[0];
1545
1546                        result = Maths.subtract(a, amin[0]);
1547                        result.idivide(aptp);
1548                } else {
1549                        double[] aptp = new double[is];
1550                        for (int j = 0; j < is; j++) {
1551                                aptp[j] = amax[j] - amin[j];
1552                        }
1553
1554                        result = Maths.subtract(a, amin);
1555                        result.idivide(aptp);
1556                }
1557                return (CompoundDataset) result;
1558        }
1559
1560        /**
1561         * Function that returns a normalised dataset which is bounded between 0 and 1
1562         * and has been distributed on a log10 scale
1563         * @param a dataset
1564         * @return normalised dataset
1565         */
1566        public static Dataset lognorm(Dataset a) {
1567                double amin = a.min().doubleValue();
1568                double aptp = Math.log10(a.max().doubleValue() - amin + 1.);
1569                Dataset temp = Maths.subtract(a, amin - 1.);
1570                temp = Maths.log10(temp);
1571                temp = Maths.divide(temp, aptp);
1572                return temp;
1573        }
1574
1575        /**
1576         * Function that returns a normalised dataset which is bounded between 0 and 1
1577         * and has been distributed on a natural log scale
1578         * @param a dataset
1579         * @return normalised dataset
1580         */
1581        public static Dataset lnnorm(Dataset a) {
1582                double amin = a.min().doubleValue();
1583                double aptp = Math.log(a.max().doubleValue() - amin + 1.);
1584                Dataset temp = Maths.subtract(a, amin - 1.);
1585                temp = Maths.log(temp);
1586                temp = Maths.divide(temp, aptp);
1587                return temp;
1588        }
1589
1590        /**
1591         * Construct a list of datasets where each represents a coordinate varying over the hypergrid
1592         * formed by the input list of axes
1593         *
1594         * @param axes an array of 1D datasets representing axes
1595         * @return a list of coordinate datasets
1596         */
1597        public static List<Dataset> meshGrid(final Dataset... axes) {
1598                List<Dataset> result = new ArrayList<Dataset>();
1599                int rank = axes.length;
1600
1601                if (rank < 2) {
1602                        utilsLogger.error("Two or more axes datasets are required");
1603                        throw new IllegalArgumentException("Two or more axes datasets are required");
1604                }
1605
1606                int[] nshape = new int[rank];
1607
1608                for (int i = 0; i < rank; i++) {
1609                        Dataset axis = axes[i];
1610                        if (axis.getRank() != 1) {
1611                                utilsLogger.error("Given axis is not 1D");
1612                                throw new IllegalArgumentException("Given axis is not 1D");
1613                        }
1614                        nshape[i] = axis.getSize();
1615                }
1616
1617                for (int i = 0; i < rank; i++) {
1618                        Dataset axis = axes[i];
1619                        Dataset coord = DatasetFactory.zeros(axis.getClass(), nshape);
1620                        result.add(coord);
1621
1622                        final int alen = axis.getSize();
1623                        for (int j = 0; j < alen; j++) {
1624                                final Object obj = axis.getObjectAbs(j);
1625                                PositionIterator pi = coord.getPositionIterator(i);
1626                                final int[] pos = pi.getPos();
1627
1628                                pos[i] = j;
1629                                while (pi.hasNext()) {
1630                                        coord.set(obj, pos);
1631                                }
1632                        }
1633                }
1634
1635                return result;
1636        }
1637
1638        /**
1639         * Generate an index dataset for given dataset shape where sub-datasets contain index values
1640         *
1641         * @param shape for indexing
1642         * @return an index dataset
1643         */
1644        public static IntegerDataset indices(int... shape) {
1645                // now create another dataset to plot against
1646                final int rank = shape.length;
1647                int[] nshape = new int[rank+1];
1648                nshape[0] = rank;
1649                for (int i = 0; i < rank; i++) {
1650                        nshape[i+1] = shape[i];
1651                }
1652
1653                IntegerDataset index = new IntegerDataset(nshape);
1654
1655                if (rank == 1) {
1656                        final int alen = shape[0];
1657                        int[] pos = new int[2];
1658                        for (int j = 0; j < alen; j++) {
1659                                pos[1] = j;
1660                                index.set(j, pos);
1661                        }
1662                } else {
1663                        for (int i = 1; i <= rank; i++) {
1664                                final int alen = nshape[i];
1665                                for (int j = 0; j < alen; j++) {
1666                                        PositionIterator pi = index.getPositionIterator(0, i);
1667                                        final int[] pos = pi.getPos();
1668
1669                                        pos[0] = i-1;
1670                                        pos[i] = j;
1671                                        while (pi.hasNext()) {
1672                                                index.set(j, pos);
1673                                        }
1674                                }
1675                        }
1676                }
1677                return index;
1678        }
1679
1680        /**
1681         * Get the centroid value of a dataset, this function works out the centroid in every direction
1682         *
1683         * @param a
1684         *            the dataset to be analysed
1685         * @param bases the optional array of base coordinates to use as weights.
1686         * This defaults to the mid-point of indices
1687         * @return a double array containing the centroid for each dimension
1688         */
1689        public static double[] centroid(Dataset a, Dataset... bases) {
1690                int rank = a.getRank();
1691                if (bases.length > 0 && bases.length != rank) {
1692                        throw new IllegalArgumentException("Number of bases must be zero or match rank of dataset");
1693                }
1694
1695                int[] shape = a.getShapeRef();
1696                if (bases.length == rank) {
1697                        for (int i = 0; i < rank; i++) {
1698                                Dataset b = bases[i];
1699                                if (b.getRank() != 1 && b.getSize() != shape[i]) {
1700                                        throw new IllegalArgumentException("A base does not have shape to match given dataset");
1701                                }
1702                        }
1703                }
1704
1705                double[] dc = new double[rank];
1706                if (rank == 0)
1707                        return dc;
1708
1709                final PositionIterator iter = new PositionIterator(shape);
1710                final int[] pos = iter.getPos();
1711
1712                double tsum = 0.0;
1713                while (iter.hasNext()) {
1714                        double val = a.getDouble(pos);
1715                        tsum += val;
1716                        for (int d = 0; d < rank; d++) {
1717                                Dataset b = bases.length == 0 ? null : bases[d];
1718                                if (b == null) {
1719                                        dc[d] += (pos[d] + 0.5) * val;
1720                                } else {
1721                                        dc[d] += b.getElementDoubleAbs(pos[d]) * val;
1722                                }
1723                        }
1724                }
1725
1726                for (int d = 0; d < rank; d++) {
1727                        dc[d] /= tsum;
1728                }
1729                return dc;
1730        }
1731
1732        /**
1733         * Find linearly-interpolated crossing points where the given dataset crosses the given value
1734         *
1735         * @param d dataset
1736         * @param value crossing value
1737         * @return list of interpolated indices
1738         */
1739        public static List<Double> crossings(Dataset d, double value) {
1740                if (d.getRank() != 1) {
1741                        utilsLogger.error("Only 1d datasets supported");
1742                        throw new UnsupportedOperationException("Only 1d datasets supported");
1743                }
1744                List<Double> results = new ArrayList<Double>();
1745
1746                // run through all pairs of points on the line and see if value lies within
1747                IndexIterator it = d.getIterator();
1748                double y1, y2;
1749
1750                y2 = it.hasNext() ? d.getElementDoubleAbs(it.index) : 0;
1751                double x = 1;
1752                while (it.hasNext()) {
1753                        y1 = y2;
1754                        y2 = d.getElementDoubleAbs(it.index);
1755                        // check if value lies within pair [y1, y2]
1756                        if ((y1 <= value && value < y2) || (y1 > value && y2 <= value)) {
1757                                final double f = (value - y2)/(y2 - y1); // negative distance from right to left
1758                                results.add(x + f);
1759                        }
1760                        x++;
1761                }
1762                if (y2 == value) { // add end point of it intersects
1763                        results.add(x);
1764                }
1765
1766                return results;
1767        }
1768
1769        /**
1770         * Find x values of all the crossing points of the dataset with the given y value
1771         *
1772         * @param xAxis
1773         *            Dataset of the X axis that needs to be looked at
1774         * @param yAxis
1775         *            Dataset of the Y axis that needs to be looked at
1776         * @param yValue
1777         *            The y value the X values are required for
1778         * @return An list of doubles containing all the X coordinates of where the line crosses
1779         */
1780        public static List<Double> crossings(Dataset xAxis, Dataset yAxis, double yValue) {
1781                if (xAxis.getSize() > yAxis.getSize()) {
1782                        throw new IllegalArgumentException(
1783                                        "Number of values of yAxis must as least be equal to the number of values of xAxis");
1784                }
1785
1786                List<Double> results = new ArrayList<Double>();
1787
1788                List<Double> indices = crossings(yAxis, yValue);
1789
1790                for (double xi : indices) {
1791                        results.add(Maths.interpolate(xAxis, xi));
1792                }
1793                return results;
1794        }
1795
1796        /**
1797         * Function that uses the crossings function but prunes the result, so that multiple crossings within a
1798         * certain proportion of the overall range of the x values
1799         *
1800         * @param xAxis
1801         *            Dataset of the X axis
1802         * @param yAxis
1803         *            Dataset of the Y axis
1804         * @param yValue
1805         *            The y value the x values are required for
1806         * @param xRangeProportion
1807         *            The proportion of the overall x spread used to prune result
1808         * @return A list containing all the unique crossing points
1809         */
1810        public static List<Double> crossings(Dataset xAxis, Dataset yAxis, double yValue, double xRangeProportion) {
1811                // get the values found
1812                List<Double> vals = crossings(xAxis, yAxis, yValue);
1813
1814                // use the proportion to calculate the error spacing
1815                double error = xRangeProportion * xAxis.peakToPeak().doubleValue();
1816
1817                int i = 0;
1818                // now go through and check for groups of three crossings which are all
1819                // within the boundaries
1820                while (i <= vals.size() - 3) {
1821                        double v1 = Math.abs(vals.get(i) - vals.get(i + 2));
1822                        if (v1 < error) {
1823                                // these 3 points should be treated as one
1824                                // make the first point equal to the average of them all
1825                                vals.set(i + 2, ((vals.get(i) + vals.get(i + 1) + vals.get(i + 2)) / 3.0));
1826                                // remove the other offending points
1827                                vals.remove(i);
1828                                vals.remove(i);
1829                        } else {
1830                                i++;
1831                        }
1832                }
1833
1834                // once the thinning process has been completed, return the pruned list
1835                return vals;
1836        }
1837
1838        // recursive function
1839        private static void setRow(Object row, Dataset a, int... pos) {
1840                final int l = Array.getLength(row);
1841                final int rank = pos.length;
1842                final int[] npos = Arrays.copyOf(pos, rank+1);
1843                Object r;
1844                if (rank+1 < a.getRank()) {
1845                        for (int i = 0; i < l; i++) {
1846                                npos[rank] = i;
1847                                r = Array.get(row, i);
1848                                setRow(r, a, npos);
1849                        }
1850                } else {
1851                        for (int i = 0; i < l; i++) {
1852                                npos[rank] = i;
1853                                r = a.getObject(npos);
1854                                Array.set(row, i, r);
1855                        }
1856                }
1857        }
1858
1859        /**
1860         * Create Java array (of arrays) from dataset
1861         * @param a dataset
1862         * @return Java array (of arrays...)
1863         */
1864        public static Object createJavaArray(Dataset a) {
1865                if (a.getElementsPerItem() > 1) {
1866                        a = createDatasetFromCompoundDataset((CompoundDataset) a, true);
1867                }
1868                Object matrix;
1869
1870                int[] shape = a.getShapeRef();
1871                if (a instanceof BooleanDataset) {
1872                        matrix = Array.newInstance(boolean.class, shape);
1873                } else if (a instanceof ByteDataset) {
1874                        matrix = Array.newInstance(byte.class, shape);
1875                } else if (a instanceof ShortDataset) {
1876                        matrix = Array.newInstance(short.class, shape);
1877                } else if (a instanceof IntegerDataset) {
1878                        matrix = Array.newInstance(int.class, shape);
1879                } else if (a instanceof LongDataset) {
1880                        matrix = Array.newInstance(long.class, shape);
1881                } else if (a instanceof FloatDataset) {
1882                        matrix = Array.newInstance(float.class, shape);
1883                } else if (a instanceof DoubleDataset) {
1884                        matrix = Array.newInstance(double.class, shape);
1885                } else {
1886                        utilsLogger.error("Dataset type not supported");
1887                        throw new IllegalArgumentException("Dataset type not supported");
1888                }
1889                // populate matrix
1890                setRow(matrix, a);
1891                return matrix;
1892        }
1893
1894        /**
1895         * Removes NaNs and infinities from floating point datasets.
1896         * All other dataset types are ignored.
1897         *
1898         * @param a dataset
1899         * @param value replacement value
1900         */
1901        public static void removeNansAndInfinities(Dataset a, final Number value) {
1902                if (a instanceof DoubleDataset) {
1903                        final double dvalue = DTypeUtils.toReal(value);
1904                        final DoubleDataset set = (DoubleDataset) a;
1905                        final IndexIterator it = set.getIterator();
1906                        final double[] data = set.getData();
1907                        while (it.hasNext()) {
1908                                double x = data[it.index];
1909                                if (Double.isNaN(x) || Double.isInfinite(x))
1910                                        data[it.index] = dvalue;
1911                        }
1912                } else if (a instanceof FloatDataset) {
1913                        final float fvalue = (float) DTypeUtils.toReal(value);
1914                        final FloatDataset set = (FloatDataset) a;
1915                        final IndexIterator it = set.getIterator();
1916                        final float[] data = set.getData();
1917                        while (it.hasNext()) {
1918                                float x = data[it.index];
1919                                if (Float.isNaN(x) || Float.isInfinite(x))
1920                                        data[it.index] = fvalue;
1921                        }
1922                } else if (a instanceof CompoundDoubleDataset) {
1923                        final double dvalue = DTypeUtils.toReal(value);
1924                        final CompoundDoubleDataset set = (CompoundDoubleDataset) a;
1925                        final int is = set.getElementsPerItem();
1926                        final IndexIterator it = set.getIterator();
1927                        final double[] data = set.getData();
1928                        while (it.hasNext()) {
1929                                for (int j = 0; j < is; j++) {
1930                                        double x = data[it.index + j];
1931                                        if (Double.isNaN(x) || Double.isInfinite(x))
1932                                                data[it.index + j] = dvalue;
1933                                }
1934                        }
1935                } else if (a instanceof CompoundFloatDataset) {
1936                        final float fvalue = (float) DTypeUtils.toReal(value);
1937                        final CompoundFloatDataset set = (CompoundFloatDataset) a;
1938                        final int is = set.getElementsPerItem();
1939                        final IndexIterator it = set.getIterator();
1940                        final float[] data = set.getData();
1941                        while (it.hasNext()) {
1942                                for (int j = 0; j < is; j++) {
1943                                        float x = data[it.index + j];
1944                                        if (Float.isNaN(x) || Float.isInfinite(x))
1945                                                data[it.index + j] = fvalue;
1946                                }
1947                        }
1948                }
1949        }
1950
1951        /**
1952         * Make floating point datasets contain only finite values. Infinities and NaNs are replaced
1953         * by +/- MAX_VALUE and 0, respectively.
1954         * All other dataset types are ignored.
1955         *
1956         * @param a dataset
1957         */
1958        public static void makeFinite(Dataset a) {
1959                if (a instanceof DoubleDataset) {
1960                        final DoubleDataset set = (DoubleDataset) a;
1961                        final IndexIterator it = set.getIterator();
1962                        final double[] data = set.getData();
1963                        while (it.hasNext()) {
1964                                final double x = data[it.index];
1965                                if (Double.isNaN(x))
1966                                        data[it.index] = 0;
1967                                else if (Double.isInfinite(x))
1968                                        data[it.index] = x > 0 ? Double.MAX_VALUE : -Double.MAX_VALUE;
1969                        }
1970                } else if (a instanceof FloatDataset) {
1971                        final FloatDataset set = (FloatDataset) a;
1972                        final IndexIterator it = set.getIterator();
1973                        final float[] data = set.getData();
1974                        while (it.hasNext()) {
1975                                final float x = data[it.index];
1976                                if (Float.isNaN(x))
1977                                        data[it.index] = 0;
1978                                else if (Float.isInfinite(x))
1979                                        data[it.index] = x > 0 ? Float.MAX_VALUE : -Float.MAX_VALUE;
1980                        }
1981                } else if (a instanceof CompoundDoubleDataset) {
1982                        final CompoundDoubleDataset set = (CompoundDoubleDataset) a;
1983                        final int is = set.getElementsPerItem();
1984                        final IndexIterator it = set.getIterator();
1985                        final double[] data = set.getData();
1986                        while (it.hasNext()) {
1987                                for (int j = 0; j < is; j++) {
1988                                        final double x = data[it.index + j];
1989                                        if (Double.isNaN(x))
1990                                                data[it.index + j] = 0;
1991                                        else if (Double.isInfinite(x))
1992                                                data[it.index + j] = x > 0 ? Double.MAX_VALUE : -Double.MAX_VALUE;
1993                                }
1994                        }
1995                } else if (a instanceof CompoundFloatDataset) {
1996                        final CompoundFloatDataset set = (CompoundFloatDataset) a;
1997                        final int is = set.getElementsPerItem();
1998                        final IndexIterator it = set.getIterator();
1999                        final float[] data = set.getData();
2000                        while (it.hasNext()) {
2001                                for (int j = 0; j < is; j++) {
2002                                        final float x = data[it.index + j];
2003                                        if (Float.isNaN(x))
2004                                                data[it.index + j] = 0;
2005                                        else if (Float.isInfinite(x))
2006                                                data[it.index + j] = x > 0 ? Float.MAX_VALUE : -Float.MAX_VALUE;
2007                                }
2008                        }
2009                }
2010        }
2011
2012        /**
2013         * Find absolute index of first value in dataset that is equal to given number
2014         * @param a dataset
2015         * @param n value
2016         * @return absolute index (if greater than a.getSize() then no value found)
2017         */
2018        public static int findIndexEqualTo(final Dataset a, final double n) {
2019                IndexIterator iter = a.getIterator();
2020                while (iter.hasNext()) {
2021                        if (a.getElementDoubleAbs(iter.index) == n)
2022                                break;
2023                }
2024
2025                return iter.index;
2026        }
2027
2028        /**
2029         * Find absolute index of first value in dataset that is greater than given number
2030         * @param a dataset
2031         * @param n value
2032         * @return absolute index (if greater than a.getSize() then no value found)
2033         */
2034        public static int findIndexGreaterThan(final Dataset a, final double n) {
2035                IndexIterator iter = a.getIterator();
2036                while (iter.hasNext()) {
2037                        if (a.getElementDoubleAbs(iter.index) > n)
2038                                break;
2039                }
2040
2041                return iter.index;
2042        }
2043
2044        /**
2045         * Find absolute index of first value in dataset that is greater than or equal to given number
2046         * @param a dataset
2047         * @param n value
2048         * @return absolute index (if greater than a.getSize() then no value found)
2049         */
2050        public static int findIndexGreaterThanOrEqualTo(final Dataset a, final double n) {
2051                IndexIterator iter = a.getIterator();
2052                while (iter.hasNext()) {
2053                        if (a.getElementDoubleAbs(iter.index) >= n)
2054                                break;
2055                }
2056
2057                return iter.index;
2058        }
2059
2060        /**
2061         * Find absolute index of first value in dataset that is less than given number
2062         * @param a dataset
2063         * @param n value
2064         * @return absolute index (if greater than a.getSize() then no value found)
2065         */
2066        public static int findIndexLessThan(final Dataset a, final double n) {
2067                IndexIterator iter = a.getIterator();
2068                while (iter.hasNext()) {
2069                        if (a.getElementDoubleAbs(iter.index) < n)
2070                                break;
2071                }
2072
2073                return iter.index;
2074        }
2075
2076        /**
2077         * Find absolute index of first value in dataset that is less than or equal to given number
2078         * @param a dataset
2079         * @param n value
2080         * @return absolute index (if greater than a.getSize() then no value found)
2081         */
2082        public static int findIndexLessThanOrEqualTo(final Dataset a, final double n) {
2083                IndexIterator iter = a.getIterator();
2084                while (iter.hasNext()) {
2085                        if (a.getElementDoubleAbs(iter.index) <= n)
2086                                break;
2087                }
2088
2089                return iter.index;
2090        }
2091
2092        /**
2093         * Find first occurrences in one dataset of values given in another sorted dataset
2094         * @param a dataset
2095         * @param values sorted 1D dataset of values to find
2096         * @return absolute indexes of those first occurrences (-1 is used to indicate value not found)
2097         */
2098        public static IntegerDataset findFirstOccurrences(final Dataset a, final Dataset values) {
2099                if (values.getRank() != 1) {
2100                        throw new IllegalArgumentException("Values dataset must be 1D");
2101                }
2102                IntegerDataset indexes = new IntegerDataset(values.getSize());
2103                indexes.fill(-1);
2104
2105                IndexIterator it = a.getIterator();
2106                final int n = values.getSize();
2107                if (values instanceof LongDataset) {
2108                        while (it.hasNext()) {
2109                                long x = a.getElementLongAbs(it.index);
2110
2111                                int l = 0; // binary search to find value in sorted dataset
2112                                long vl = values.getLong(l);
2113                                if (x <= vl) {
2114                                        if (x == vl && indexes.getAbs(l) < 0)
2115                                                indexes.setAbs(l, it.index);
2116                                        continue;
2117                                }
2118                                int h = n - 1;
2119                                long vh = values.getLong(h);
2120                                if (x >= vh) {
2121                                        if (x == vh && indexes.getAbs(h) < 0)
2122                                                indexes.setAbs(h, it.index);
2123                                        continue;
2124                                }
2125                                while (h - l > 1) {
2126                                        int m = (l + h) / 2;
2127                                        long vm = values.getLong(m);
2128                                        if (x < vm) {
2129                                                h = m;
2130                                        } else if (x > vm) {
2131                                                l = m;
2132                                        } else {
2133                                                if (indexes.getAbs(m) < 0)
2134                                                        indexes.setAbs(m, it.index);
2135                                                break;
2136                                        }
2137                                }
2138                        }
2139                } else {
2140                        while (it.hasNext()) {
2141                                double x = a.getElementDoubleAbs(it.index);
2142
2143                                int l = 0; // binary search to find value in sorted dataset
2144                                double vl = values.getDouble(l);
2145                                if (x <= vl) {
2146                                        if (x == vl && indexes.getAbs(l) < 0)
2147                                                indexes.setAbs(l, it.index);
2148                                        continue;
2149                                }
2150                                int h = n - 1;
2151                                double vh = values.getDouble(h);
2152                                if (x >= vh) {
2153                                        if (x == vh && indexes.getAbs(h) < 0)
2154                                                indexes.setAbs(h, it.index);
2155                                        continue;
2156                                }
2157                                while (h - l > 1) {
2158                                        int m = (l + h) / 2;
2159                                        double vm = values.getDouble(m);
2160                                        if (x < vm) {
2161                                                h = m;
2162                                        } else if (x > vm) {
2163                                                l = m;
2164                                        } else {
2165                                                if (indexes.getAbs(m) < 0)
2166                                                        indexes.setAbs(m, it.index);
2167                                                break;
2168                                        }
2169                                }
2170                        }
2171                }
2172                return indexes;
2173        }
2174
2175        /**
2176         * Find indexes in sorted dataset of values for each value in other dataset
2177         * @param a dataset
2178         * @param values sorted 1D dataset of values to find
2179         * @return absolute indexes of values (-1 is used to indicate value not found)
2180         */
2181        public static IntegerDataset findIndexesForValues(final Dataset a, final Dataset values) {
2182                if (values.getRank() != 1) {
2183                        throw new IllegalArgumentException("Values dataset must be 1D");
2184                }
2185                IntegerDataset indexes = new IntegerDataset(a.getSize());
2186                indexes.fill(-1);
2187
2188                IndexIterator it = a.getIterator();
2189                int i = -1;
2190                final int n = values.getSize();
2191                if (values instanceof LongDataset) {
2192                        while (it.hasNext()) {
2193                                i++;
2194                                long x = a.getElementLongAbs(it.index);
2195
2196                                int l = 0; // binary search to find value in sorted dataset
2197                                long vl = values.getLong(l);
2198                                if (x <= vl) {
2199                                        if (x == vl)
2200                                                indexes.setAbs(i, l);
2201                                        continue;
2202                                }
2203                                int h = n - 1;
2204                                long vh = values.getLong(h);
2205                                if (x >= vh) {
2206                                        if (x == vh)
2207                                                indexes.setAbs(i, h);
2208                                        continue;
2209                                }
2210                                while (h - l > 1) {
2211                                        int m = (l + h) / 2;
2212                                        long vm = values.getLong(m);
2213                                        if (x < vm) {
2214                                                h = m;
2215                                        } else if (x > vm) {
2216                                                l = m;
2217                                        } else {
2218                                                indexes.setAbs(i, m);
2219                                                break;
2220                                        }
2221                                }
2222                        }
2223                } else {
2224                        while (it.hasNext()) {
2225                                i++;
2226                                double x = a.getElementDoubleAbs(it.index);
2227
2228                                int l = 0; // binary search to find value in sorted dataset
2229                                double vl = values.getDouble(l);
2230                                if (x <= vl) {
2231                                        if (x == vl)
2232                                                indexes.setAbs(i, l);
2233                                        continue;
2234                                }
2235                                int h = n - 1;
2236                                double vh = values.getDouble(h);
2237                                if (x >= vh) {
2238                                        if (x == vh)
2239                                                indexes.setAbs(i, h);
2240                                        continue;
2241                                }
2242                                while (h - l > 1) {
2243                                        int m = (l + h) / 2;
2244                                        double vm = values.getDouble(m);
2245                                        if (x < vm) {
2246                                                h = m;
2247                                        } else if (x > vm) {
2248                                                l = m;
2249                                        } else {
2250                                                indexes.setAbs(i, m);
2251                                                break;
2252                                        }
2253                                }
2254                        }
2255                }
2256
2257                return indexes;
2258        }
2259
2260        /**
2261         * Roll items over given axis by given amount
2262         * @param <T> dataset class
2263         * @param a dataset
2264         * @param shift amount to shift
2265         * @param axis if null, then roll flattened dataset
2266         * @return rolled dataset
2267         */
2268        public static <T extends Dataset> T roll(final T a, final int shift, Integer axis) {
2269                T r = DatasetFactory.zeros(a);
2270                int is = a.getElementsPerItem();
2271                if (axis == null) {
2272                        IndexIterator it = a.getIterator();
2273                        int s = r.getSize();
2274                        int i = shift % s;
2275                        if (i < 0)
2276                                i += s;
2277                        while (it.hasNext()) {
2278                                r.setObjectAbs(i, a.getObjectAbs(it.index));
2279                                i += is;
2280                                if (i >= s) {
2281                                        i %= s;
2282                                }
2283                        }
2284                } else {
2285                        axis = a.checkAxis(axis);
2286                        PositionIterator pi = a.getPositionIterator(axis);
2287                        int s = a.getShapeRef()[axis];
2288                        Dataset u = DatasetFactory.zeros(is, a.getClass(), new int[] {s});
2289                        Dataset v = DatasetFactory.zeros(u);
2290                        int[] pos = pi.getPos();
2291                        boolean[] hit = pi.getOmit();
2292                        while (pi.hasNext()) {
2293                                a.copyItemsFromAxes(pos, hit, u);
2294                                int i = shift % s;
2295                                if (i < 0)
2296                                        i += s;
2297                                for (int j = 0; j < s; j++) {
2298                                        v.setObjectAbs(i, u.getObjectAbs(j*is));
2299                                        i += is;
2300                                        if (i >= s) {
2301                                                i %= s;
2302                                        }
2303                                }
2304                                r.setItemsOnAxes(pos, hit, v.getBuffer());
2305                        }
2306                }
2307                return r;
2308        }
2309
2310        /**
2311         * Roll the specified axis backwards until it lies in given position
2312         * @param <T> dataset class
2313         * @param a dataset
2314         * @param axis The rolled axis (index in shape array). Other axes are left unchanged in relative positions
2315         * @param start The position with it right of the destination of the rolled axis
2316         * @return dataset with rolled axis
2317         */
2318        @SuppressWarnings("unchecked")
2319        public static <T extends Dataset> T rollAxis(final T a, int axis, int start) {
2320                int r = a.getRank();
2321                axis = a.checkAxis(axis);
2322                if (start < 0)
2323                        start += r;
2324                if (start < 0 || start > r) {
2325                        throw new IllegalArgumentException("Start is out of range: it should be >= 0 and <= " + r);
2326                }
2327                if (axis < start)
2328                        start--;
2329
2330                if (axis == start)
2331                        return a;
2332
2333                ArrayList<Integer> axes = new ArrayList<Integer>();
2334                for (int i = 0; i < r; i++) {
2335                        if (i != axis) {
2336                                axes.add(i);
2337                        }
2338                }
2339                axes.add(start, axis);
2340                int[] aa = new int[r];
2341                for (int i = 0; i < r; i++) {
2342                        aa[i] = axes.get(i);
2343                }
2344                return (T) a.getTransposedView(aa);
2345        }
2346
2347        private static SliceND createFlippedSlice(final Dataset a, int axis) {
2348                int[] shape = a.getShapeRef();
2349                SliceND slice = new SliceND(shape);
2350                slice.flip(axis);
2351                return slice;
2352        }
2353
2354        /**
2355         * Flip items in left/right direction, column-wise, or along second axis
2356         * @param <T> dataset class
2357         * @param a dataset must be at least 2D
2358         * @return view of flipped dataset
2359         */
2360        @SuppressWarnings("unchecked")
2361        public static <T extends Dataset> T flipLeftRight(final T a) {
2362                if (a.getRank() < 2) {
2363                        throw new IllegalArgumentException("Dataset must be at least 2D");
2364                }
2365                return (T) a.getSliceView(createFlippedSlice(a, 1));
2366        }
2367
2368        /**
2369         * Flip items in up/down direction, row-wise, or along first axis
2370         * @param <T> dataset class
2371         * @param a dataset
2372         * @return view of flipped dataset
2373         */
2374        @SuppressWarnings("unchecked")
2375        public static <T extends Dataset> T flipUpDown(final T a) {
2376                return (T) a.getSliceView(createFlippedSlice(a, 0));
2377        }
2378
2379        /**
2380         * Rotate items in first two dimension by 90 degrees anti-clockwise
2381         * @param <T> dataset class
2382         * @param a dataset must be at least 2D
2383         * @return view of flipped dataset
2384         */
2385        public static <T extends Dataset> T rotate90(final T a) {
2386                return rotate90(a, 1);
2387        }
2388
2389        /**
2390         * Rotate items in first two dimension by 90 degrees anti-clockwise
2391         * @param <T> dataset class
2392         * @param a dataset must be at least 2D
2393         * @param k number of 90-degree rotations
2394         * @return view of flipped dataset
2395         */
2396        @SuppressWarnings("unchecked")
2397        public static <T extends Dataset> T rotate90(final T a, int k) {
2398                k = k % 4;
2399                while (k < 0) {
2400                        k += 4;
2401                }
2402                int r = a.getRank();
2403                if (r < 2) {
2404                        throw new IllegalArgumentException("Dataset must be at least 2D");
2405                }
2406                switch (k) {
2407                case 1: case 3:
2408                        int[] axes = new int[r];
2409                        axes[0] = 1;
2410                        axes[1] = 0;
2411                        for (int i = 2; i < r; i++) {
2412                                axes[i] = i;
2413                        }
2414                        Dataset t = a.getTransposedView(axes);
2415                        return (T) t.getSliceView(createFlippedSlice(t, k == 1 ? 0 : 1));
2416                case 2:
2417                        SliceND s = createFlippedSlice(a, 0);
2418                        s.flip(1);
2419                        return (T) a.getSliceView(s);
2420                default:
2421                case 0:
2422                        return a;
2423                }
2424        }
2425
2426        /**
2427         * Select content according where condition is true. All inputs are broadcasted to a maximum shape
2428         * @param condition boolean dataset
2429         * @param x first input
2430         * @param y second input
2431         * @return dataset where content is x or y depending on whether condition is true or otherwise
2432         */
2433        public static Dataset select(BooleanDataset condition, Object x, Object y) {
2434                Object[] all = new Object[] {condition, x, y};
2435                Dataset[] dAll = BroadcastUtils.convertAndBroadcast(all);
2436                condition = (BooleanDataset) dAll[0];
2437                Dataset dx = dAll[1];
2438                Dataset dy = dAll[2];
2439                Class<? extends Dataset> dc = InterfaceUtils.getBestInterface(dx.getClass(), dy.getClass());
2440                int ds = Math.max(dx.getElementsPerItem(), dy.getElementsPerItem());
2441
2442                Dataset r = DatasetFactory.zeros(ds, dc, condition.getShapeRef());
2443                IndexIterator iter = condition.getIterator(true);
2444                final int[] pos = iter.getPos();
2445                int i = 0;
2446                while (iter.hasNext()) {
2447                        r.setObjectAbs(i++, condition.getElementBooleanAbs(iter.index) ? dx.getObject(pos) : dy.getObject(pos));
2448                }
2449                return r;
2450        }
2451
2452        /**
2453         * Select content from choices where condition is true, otherwise use default. All inputs are broadcasted to a maximum shape
2454         * @param conditions array of boolean datasets
2455         * @param choices array of datasets or objects
2456         * @param def default value (can be a dataset)
2457         * @return dataset
2458         */
2459        public static Dataset select(BooleanDataset[] conditions, Object[] choices, Object def) {
2460                final int n = conditions.length;
2461                if (choices.length != n) {
2462                        throw new IllegalArgumentException("Choices list is not same length as conditions list");
2463                }
2464                Object[] all = new Object[2*n];
2465                System.arraycopy(conditions, 0, all, 0, n);
2466                System.arraycopy(choices, 0, all, n, n);
2467                Dataset[] dAll = BroadcastUtils.convertAndBroadcast(all);
2468                conditions = new BooleanDataset[n];
2469                Dataset[] dChoices = new Dataset[n];
2470                System.arraycopy(dAll, 0, conditions, 0, n);
2471                System.arraycopy(dAll, n, dChoices, 0, n);
2472                Class<? extends Dataset> dc = null;
2473                int ds = -1;
2474                for (int i = 0; i < n; i++) {
2475                        Dataset a = dChoices[i];
2476                        Class<? extends Dataset> c = a.getClass();
2477                        dc = InterfaceUtils.getBestInterface(dc, c);
2478                        int s = a.getElementsPerItem();
2479                        if (s > ds) {
2480                                ds = s;
2481                        }
2482                }
2483                if (dc == null || ds < 1) {
2484                        throw new IllegalArgumentException("Dataset types of choices are invalid");
2485                }
2486
2487                Dataset r = DatasetFactory.zeros(ds, dc, conditions[0].getShapeRef());
2488                Dataset d = DatasetFactory.createFromObject(def).getBroadcastView(r.getShapeRef());
2489                PositionIterator iter = new PositionIterator(r.getShapeRef());
2490                final int[] pos = iter.getPos();
2491                int i = 0;
2492                while (iter.hasNext()) {
2493                        int j = 0;
2494                        for (; j < n; j++) {
2495                                if (conditions[j].get(pos)) {
2496                                        r.setObjectAbs(i++, dChoices[j].getObject(pos));
2497                                        break;
2498                                }
2499                        }
2500                        if (j == n) {
2501                                r.setObjectAbs(i++, d.getObject(pos));
2502                        }
2503                }
2504                return r;
2505        }
2506
2507        /**
2508         * Choose content from choices where condition is true, otherwise use default. All inputs are broadcasted to a maximum shape
2509         * @param index integer dataset (ideally, items should be in [0, n) range, if there are n choices)
2510         * @param choices array of datasets or objects
2511         * @param throwAIOOBE if true, throw array index out of bound exception
2512         * @param clip true to clip else wrap indices out of bounds; only used when throwAOOBE is false
2513         * @return dataset
2514         */
2515        public static Dataset choose(IntegerDataset index, Object[] choices, boolean throwAIOOBE, boolean clip) {
2516                final int n = choices.length;
2517                Object[] all = new Object[n + 1];
2518                System.arraycopy(choices, 0, all, 0, n);
2519                all[n] = index;
2520                Dataset[] dChoices = BroadcastUtils.convertAndBroadcast(all);
2521                Class<? extends Dataset> dc = null;
2522                int ds = -1;
2523                int mr = -1;
2524                for (int i = 0; i < n; i++) {
2525                        Dataset a = dChoices[i];
2526                        int r = a.getRank();
2527                        if (r > mr) {
2528                                mr = r;
2529                        }
2530                        dc = InterfaceUtils.getBestInterface(dc, a.getClass());
2531                        int s = a.getElementsPerItem();
2532                        if (s > ds) {
2533                                ds = s;
2534                        }
2535                }
2536                if (dc == null || ds < 1) {
2537                        throw new IllegalArgumentException("Dataset types of choices are invalid");
2538                }
2539                index = (IntegerDataset) dChoices[n];
2540                dChoices[n] = null;
2541
2542                Dataset r = DatasetFactory.zeros(ds, dc, index.getShapeRef());
2543                IndexIterator iter = index.getIterator(true);
2544                final int[] pos = iter.getPos();
2545                int i = 0;
2546                while (iter.hasNext()) {
2547                        int j = index.getAbs(iter.index);
2548                        if (j < 0) {
2549                                if (throwAIOOBE)
2550                                        throw new ArrayIndexOutOfBoundsException(j);
2551                                if (clip) {
2552                                        j = 0;
2553                                } else {
2554                                        j %= n;
2555                                        j += n; // as remainder still negative
2556                                }
2557                        }
2558                        if (j >= n) {
2559                                if (throwAIOOBE)
2560                                        throw new ArrayIndexOutOfBoundsException(j);
2561                                if (clip) {
2562                                        j = n - 1;
2563                                } else {
2564                                        j %= n;
2565                                }
2566                        }
2567                        Dataset c = dChoices[j];
2568                        r.setObjectAbs(i++, c.getObject(pos));
2569                }
2570                return r;
2571        }
2572
2573        /**
2574         * Calculate positions in given shape from a dataset of 1-D indexes
2575         * @param indices dataset values taken as integers for index
2576         * @param shape dataset shape
2577         * @return list of positions as integer datasets
2578         */
2579        public static List<IntegerDataset> calcPositionsFromIndexes(Dataset indices, int[] shape) {
2580                int rank = shape.length;
2581                List<IntegerDataset> posns = new ArrayList<IntegerDataset>();
2582                int[] iShape = indices.getShapeRef();
2583                for (int i = 0; i < rank; i++) {
2584                        posns.add(new IntegerDataset(iShape));
2585                }
2586                IndexIterator it = indices.getIterator(true);
2587                int[] pos = it.getPos();
2588                while (it.hasNext()) {
2589                        int n = indices.getInt(pos);
2590                        int[] p = ShapeUtils.getNDPositionFromShape(n, shape);
2591                        for (int i = 0; i < rank; i++) {
2592                                posns.get(i).setItem(p[i], pos);
2593                        }
2594                }
2595                return posns;
2596        }
2597
2598
2599        /**
2600         * Calculate indexes in given shape from datasets of position
2601         * @param positions as a list of datasets where each holds the position in a dimension
2602         * @param shape dataset shape
2603         * @param mode either null, zero-length, unit length or length of rank of shape where
2604         *  0 = raise exception, 1 = wrap, 2 = clip
2605         * @return indexes as an integer dataset
2606         */
2607        public static IntegerDataset calcIndexesFromPositions(List<? extends Dataset> positions, int[] shape, int... mode) {
2608                int rank = shape.length;
2609                if (positions.size() != rank) {
2610                        throw new IllegalArgumentException("Number of position datasets must be equal to rank of shape");
2611                }
2612
2613                if (mode == null || mode.length == 0) {
2614                        mode = new int[rank];
2615                } else if (mode.length == 1) {
2616                        int m = mode[0];
2617                        mode = new int[rank];
2618                        Arrays.fill(mode, m);
2619                } else if (mode.length != rank) {
2620                        throw new IllegalArgumentException("Mode length greater than one must match rank of shape");
2621                }
2622                for (int i = 0; i < rank; i++) {
2623                        int m = mode[i];
2624                        if (m < 0 || m > 2) {
2625                                throw new IllegalArgumentException("Unknown mode value - it must be 0, 1, or 2");
2626                        }
2627                }
2628
2629                Dataset p = positions.get(0);
2630                IntegerDataset indexes = new IntegerDataset(p.getShapeRef());
2631                IndexIterator it = p.getIterator(true);
2632                int[] iPos = it.getPos();
2633                int[] tPos = new int[rank];
2634                while (it.hasNext()) {
2635                        for (int i = 0; i < rank; i++) {
2636                                p = positions.get(i);
2637                                int j = p.getInt(iPos);
2638                                int d = shape[i];
2639                                if (mode[i] == 0) {
2640                                        if (j < 0 || j >= d) {
2641                                                throw new ArrayIndexOutOfBoundsException("Position value exceeds dimension in shape");
2642                                        }
2643                                } else if (mode[i] == 1) {
2644                                        while (j < 0)
2645                                                j += d;
2646                                        while (j >= d)
2647                                                j -= d;
2648                                } else {
2649                                        if (j < 0)
2650                                                j = 0;
2651                                        if (j >= d)
2652                                                j = d - 1;
2653                                }
2654                                tPos[i] = j;
2655                        }
2656                        indexes.set(ShapeUtils.getFlat1DIndex(shape, tPos), iPos);
2657                }
2658
2659                return indexes;
2660        }
2661
2662        /**
2663         * Serialize dataset by flattening it. Discards metadata
2664         * @param data dataset
2665         * @return some java array
2666         */
2667        public static Serializable serializeDataset(final IDataset data) {
2668                Dataset d = convertToDataset(data).getView(false);
2669                d.clearMetadata(null);
2670                return d.flatten().getBuffer();
2671        }
2672
2673        /**
2674         * Extract values where condition is non-zero. This is similar to Dataset#getByBoolean but supports broadcasting
2675         * @param data dataset
2676         * @param condition should be broadcastable to data
2677         * @return 1-D dataset of values
2678         */
2679        public static Dataset extract(final IDataset data, final IDataset condition) {
2680                Dataset a = convertToDataset(data.getSliceView());
2681                Dataset b = cast(BooleanDataset.class, condition.getSliceView());
2682
2683                try {
2684                        return a.getByBoolean(b);
2685                } catch (IllegalArgumentException e) {
2686                        final int length = ((Number) b.sum()).intValue();
2687
2688                        BroadcastPairIterator it = new BroadcastPairIterator(a, b, null, false);
2689                        int size = ShapeUtils.calcSize(it.getShape());
2690                        Dataset c;
2691                        if (length < size) {
2692                                int[] ashape = it.getFirstShape();
2693                                int[] bshape = it.getSecondShape();
2694                                int r = ashape.length;
2695                                size = length;
2696                                for (int i = 0; i < r; i++) {
2697                                        int s = ashape[i];
2698                                        if (s > 1 && bshape[i] == 1) {
2699                                                size *= s;
2700                                        }
2701                                }
2702                        }
2703                        c = DatasetFactory.zeros(a.getClass(), size);
2704
2705                        int i = 0;
2706                        if (it.isOutputDouble()) {
2707                                while (it.hasNext()) {
2708                                        if (it.bLong != 0) {
2709                                                c.setObjectAbs(i++, it.aDouble);
2710                                        }
2711                                }
2712                        } else {
2713                                while (it.hasNext()) {
2714                                        if (it.bLong != 0) {
2715                                                c.setObjectAbs(i++, it.aLong);
2716                                        }
2717                                }
2718                        }
2719
2720                        return c;
2721                }
2722        }
2723
2724        /**
2725         * Set shape to keep original rank
2726         * @param a dataset
2727         * @param originalShape original shape
2728         * @param axes dimensions in original shape to set to 1
2729         * @since 2.2
2730         */
2731        public static void setShapeToOriginalRank(ILazyDataset a, int[] originalShape, int... axes) {
2732                a.setShape(ShapeUtils.getReducedShapeKeepRank(originalShape, axes));
2733        }
2734}