import me.lemire.integercompression.*;

import me.lemire.integercompression.differential.*;

import java.util.*;

public class example {

public static void main(String[] args) {

superSimpleExample();

unsortedExample();

basicExample();

basicExampleHeadless();

advancedExample();

headlessDemo();

}

public static void superSimpleExample() {

IntegratedIntCompressor iic = new IntegratedIntCompressor();

int[] data = new int[2342351];

for(int k = 0; k < data.length; ++k)

data[k] = k;

System.out.println("Compressing "+data.length+" integers using friendly interface");

int[] compressed = iic.compress(data);

int[] recov = iic.uncompress(compressed);

System.out.println("compressed from "+data.length*4/1024+"KB to "+compressed.length*4/1024+"KB");

if(!Arrays.equals(recov,data)) throw new RuntimeException("bug");

}

public static void basicExample() {

int[] data = new int[2342351];

System.out.println("Compressing "+data.length+" integers in one go");

// data should be sorted for best

//results

for(int k = 0; k < data.length; ++k)

data[k] = k;

// Very important: the data is in sorted order!!! If not, you

// will get very poor compression with IntegratedBinaryPacking,

// you should use another CODEC.

// next we compose a CODEC. Most of the processing

// will be done with binary packing, and leftovers will

// be processed using variable byte

IntegratedIntegerCODEC codec = new

IntegratedComposition(

new IntegratedBinaryPacking(),

new IntegratedVariableByte());

// output vector should be large enough...

int [] compressed = new int[data.length+1024];

// compressed might not be large enough in some cases

// if you get java.lang.ArrayIndexOutOfBoundsException, try

// allocating more memory

/**

*

* compressing

*

*/

IntWrapper inputoffset = new IntWrapper(0);

IntWrapper outputoffset = new IntWrapper(0);

codec.compress(data,inputoffset,data.length,compressed,outputoffset);

// got it!

// inputoffset should be at data.length but outputoffset tells

// us where we are...

System.out.println("compressed from "+data.length*4/1024+"KB to "+outputoffset.intValue()*4/1024+"KB");

// we can repack the data: (optional)

compressed = Arrays.copyOf(compressed,outputoffset.intValue());

/**

*

* now uncompressing

*

* This assumes that we otherwise know how many integers

* have been compressed, or we can bound it (e.g., you know that

* will never need to decore more than 2000 integers).

* See basicExampleHeadless for a

* more general case where you can manually manage the compressed

* array size.

*/

int[] recovered = new int[data.length];

IntWrapper recoffset = new IntWrapper(0);

codec.uncompress(compressed,new IntWrapper(0),compressed.length,recovered,recoffset);

if(Arrays.equals(data,recovered))

System.out.println("data is recovered without loss");

else

throw new RuntimeException("bug"); // could use assert

System.out.println();

}

/**

* Like the basicExample, but we store the input array size manually.

*/

public static void basicExampleHeadless() {

int[] data = new int[2342351];

System.out.println("Compressing " + data.length + " integers in one go using the headless approach");

// data should be sorted for best

// results

for (int k = 0; k < data.length; ++k)

data[k] = k;

// Very important: the data is in sorted order!!! If not, you

// will get very poor compression with IntegratedBinaryPacking,

// you should use another CODEC.

// next we compose a CODEC. Most of the processing

// will be done with binary packing, and leftovers will

// be processed using variable byte

SkippableIntegratedComposition codec = new SkippableIntegratedComposition(new IntegratedBinaryPacking(),

new IntegratedVariableByte());

int[] compressed = new int[codec.maxHeadlessCompressedLength(new IntWrapper(0), data.length)];

/**

*

* compressing

*

*/

IntWrapper inputoffset = new IntWrapper(0);

IntWrapper outputoffset = new IntWrapper(1);

compressed[0] = data.length; // we manually store how many integers

codec.headlessCompress(data, inputoffset, data.length, compressed, outputoffset, new IntWrapper(0));

// got it!

// inputoffset should be at data.length but outputoffset tells

// us where we are...

System.out.println(

"compressed from " + data.length * 4 / 1024 + "KB to " + outputoffset.intValue() * 4 / 1024 + "KB");

// we can repack the data: (optional)

compressed = Arrays.copyOf(compressed, outputoffset.intValue());

/**

*

* now uncompressing

*

*/

int howmany = compressed[0];// we manually stored the number of

// compressed integers

int[] recovered = new int[howmany];

IntWrapper recoffset = new IntWrapper(0);

codec.headlessUncompress(compressed, new IntWrapper(1), compressed.length, recovered, recoffset, howmany, new IntWrapper(0));

if (Arrays.equals(data, recovered))

System.out.println("data is recovered without loss");

else

throw new RuntimeException("bug"); // could use assert

System.out.println();

}

/**

* This is an example to show you can compress unsorted integers

* as long as most are small.

*/

public static void unsortedExample() {

final int N = 1333333;

int[] data = new int[N];

// initialize the data (most will be small

for(int k = 0; k < N; k+=1) data[k] = 3;

// throw some larger values

for(int k = 0; k < N; k+=5) data[k] = 100;

for(int k = 0; k < N; k+=533) data[k] = 10000;

int[] compressed = new int [N+1024];// could need more

IntegerCODEC codec = new

Composition(

new FastPFOR(),

new VariableByte());

// compressing

IntWrapper inputoffset = new IntWrapper(0);

IntWrapper outputoffset = new IntWrapper(0);

codec.compress(data,inputoffset,data.length,compressed,outputoffset);

System.out.println("compressed unsorted integers from "+data.length*4/1024+"KB to "+outputoffset.intValue()*4/1024+"KB");

// we can repack the data: (optional)

compressed = Arrays.copyOf(compressed,outputoffset.intValue());

int[] recovered = new int[N];

IntWrapper recoffset = new IntWrapper(0);

codec.uncompress(compressed,new IntWrapper(0),compressed.length,recovered,recoffset);

if(Arrays.equals(data,recovered))

System.out.println("data is recovered without loss");

else

throw new RuntimeException("bug"); // could use assert

System.out.println();

}

/**

* This is like the basic example, but we

* show how to process larger arrays in chunks.

*

* Some of this code was written by Pavel Klinov.

*/

public static void advancedExample() {

int TotalSize = 2342351; // some arbitrary number

int ChunkSize = 16384; // size of each chunk, choose a multiple of 128

System.out.println("Compressing "+TotalSize+" integers using chunks of "+ChunkSize+" integers ("+ChunkSize*4/1024+"KB)");

System.out.println("(It is often better for applications to work in chunks fitting in CPU cache.)");

int[] data = new int[TotalSize];

// data should be sorted for best

//results

for(int k = 0; k < data.length; ++k)

data[k] = k;

// next we compose a CODEC. Most of the processing

// will be done with binary packing, and leftovers will

// be processed using variable byte, using variable byte

// only for the last chunk!

IntegratedIntegerCODEC regularcodec = new

IntegratedBinaryPacking();

IntegratedVariableByte ivb = new IntegratedVariableByte();

IntegratedIntegerCODEC lastcodec = new

IntegratedComposition(regularcodec,ivb);

// output vector should be large enough...

int [] compressed = new int[TotalSize+1024];

/**

*

* compressing

*

*/

IntWrapper inputoffset = new IntWrapper(0);

IntWrapper outputoffset = new IntWrapper(0);

for(int k = 0; k < TotalSize / ChunkSize; ++k)

regularcodec.compress(data,inputoffset,ChunkSize,compressed,outputoffset);

lastcodec.compress(data, inputoffset, TotalSize % ChunkSize, compressed, outputoffset);

// got it!

// inputoffset should be at data.length but outputoffset tells

// us where we are...

System.out.println("compressed from "+data.length*4/1024+"KB to "+outputoffset.intValue()*4/1024+"KB");

// we can repack the data:

compressed = Arrays.copyOf(compressed,outputoffset.intValue());

/**

*

* now uncompressing

*

* We are *not* assuming that the original array length is known, however

* we assume that the chunk size (ChunkSize) is known.

*

*/

int[] recovered = new int[ChunkSize];

IntWrapper compoff = new IntWrapper(0);

IntWrapper recoffset;

int currentpos = 0;

while(compoff.get()<compressed.length) {

recoffset = new IntWrapper(0);

regularcodec.uncompress(compressed,compoff,compressed.length - compoff.get(),recovered,recoffset);

if(recoffset.get() < ChunkSize) {// last chunk detected

ivb.uncompress(compressed,compoff,compressed.length - compoff.get(),recovered,recoffset);

}

for(int i = 0; i < recoffset.get(); ++i) {

if(data[currentpos+i] != recovered[i]) throw new RuntimeException("bug"); // could use assert

}

currentpos += recoffset.get();

}

System.out.println("data is recovered without loss");

System.out.println();

}

/*

* Demo of the headless approach where we must supply the array length

*/

public static void headlessDemo() {

System.out.println("Compressing arrays with minimal header...");

int[] uncompressed1 = {1,2,1,3,1};

int[] uncompressed2 = {3,2,4,6,1};

SkippableIntegerCODEC codec = new SkippableComposition(new BinaryPacking(), new VariableByte());

int maxCompressedLength = codec.maxHeadlessCompressedLength(new IntWrapper(0), uncompressed1.length)

+ codec.maxHeadlessCompressedLength(new IntWrapper(0), uncompressed2.length);

int[] compressed = new int[maxCompressedLength];

// compressing

IntWrapper outPos = new IntWrapper();

IntWrapper previous = new IntWrapper();

codec.headlessCompress(uncompressed1,new IntWrapper(),uncompressed1.length,compressed,outPos);

int length1 = outPos.get() - previous.get();

previous = new IntWrapper(outPos.get());

codec.headlessCompress(uncompressed2,new IntWrapper(),uncompressed2.length,compressed,outPos);

int length2 = outPos.get() - previous.get();

compressed = Arrays.copyOf(compressed,length1 + length2);

System.out.println("compressed unsorted integers from "+uncompressed1.length*4+"B to "+length1*4+"B");

System.out.println("compressed unsorted integers from "+uncompressed2.length*4+"B to "+length2*4+"B");

System.out.println("Total compressed output "+compressed.length);

int[] recovered1 = new int[uncompressed1.length];

int[] recovered2 = new int[uncompressed1.length];

IntWrapper inPos = new IntWrapper();

System.out.println("Decoding first array starting at pos = "+inPos);

codec.headlessUncompress(compressed,inPos, compressed.length, recovered1, new IntWrapper(0), uncompressed1.length);

System.out.println("Decoding second array starting at pos = "+inPos);

codec.headlessUncompress(compressed,inPos, compressed.length, recovered2, new IntWrapper(0), uncompressed2.length);

if(!Arrays.equals(uncompressed1,recovered1)) throw new RuntimeException("First array does not match.");

if(!Arrays.equals(uncompressed2,recovered2)) throw new RuntimeException("Second array does not match.");

System.out.println("The arrays match, your code is probably ok.");

}

}