Data manipulation

Recipes related to compare, filtering or transforming data.

Concatenate VectorSchemaRoots

In some cases, VectorSchemaRoot needs to be modeled as a container. To accomplish this, you can use VectorSchemaRootAppender.append. The following code creates two roots, then concatenates them together:

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.arrow.vector.util.VectorSchemaRootAppender;

import static java.util.Arrays.asList;

Field column_one = new Field("column-one", FieldType.nullable(new ArrowType.Int(32, true)), null);
Schema schema = new Schema(asList(column_one));
try (
    BufferAllocator allocator = new RootAllocator();
    VectorSchemaRoot rootOne = VectorSchemaRoot.create(schema, allocator);
    VectorSchemaRoot rootTwo = VectorSchemaRoot.create(schema, allocator);
    VectorSchemaRoot result = VectorSchemaRoot.create(schema, allocator);
) {
    IntVector appenderOne = (IntVector) rootOne.getVector(0);
    rootOne.allocateNew();
    appenderOne.set(0, 100);
    appenderOne.set(1, 20);
    rootOne.setRowCount(2);
    IntVector appenderTwo = (IntVector) rootTwo.getVector(0);
    rootTwo.allocateNew();
    appenderTwo.set(0, 34);
    appenderTwo.set(1, 75);
    rootTwo.setRowCount(2);
    result.allocateNew();
    VectorSchemaRootAppender.append(result, rootOne, rootTwo);
    System.out.print(result.contentToTSVString());
}
column-one
100
20
34
75

Concatenate Value Vectors

In some cases, we need to concatenate two value vectors into one. To accomplish this, we can use VectorAppender. This mutates the initial ValueVector.

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.util.VectorAppender;

try (
    BufferAllocator allocator = new RootAllocator();
    IntVector initialValues = new IntVector("initialValues", allocator);
    IntVector toAppend = new IntVector("toAppend", allocator);
) {
    initialValues.allocateNew(2);
    initialValues.set(0, 1);
    initialValues.set(1, 2);
    initialValues.setValueCount(2);
    System.out.println("Initial IntVector: " + initialValues);
    toAppend.allocateNew(4);
    toAppend.set(1, 4);
    toAppend.set(3, 6);
    toAppend.setValueCount(4);
    System.out.println("IntVector to Append: " + toAppend);
    VectorAppender appenderUtil = new VectorAppender(initialValues);
    toAppend.accept(appenderUtil, null);
    System.out.println("IntVector Result: " + initialValues);
}
Initial IntVector: [1, 2]
IntVector to Append: [null, 4, null, 6]
IntVector Result: [1, 2, null, 4, null, 6]

Compare Vectors for Field Equality

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.compare.TypeEqualsVisitor;
import org.apache.arrow.memory.RootAllocator;

try(
    BufferAllocator allocator = new RootAllocator();
    IntVector right = new IntVector("int", allocator);
) {
    right.allocateNew(3);
    right.set(0, 10);
    right.set(1, 20);
    right.set(2, 30);
    right.setValueCount(3);
    IntVector left1 = new IntVector("int", allocator);
    IntVector left2 = new IntVector("int2", allocator);
    TypeEqualsVisitor visitor = new TypeEqualsVisitor(right);

    System.out.println(visitor.equals(left1));
    System.out.println(visitor.equals(left2));
}
true
false

Compare Vectors Equality

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.compare.VectorEqualsVisitor;

try(
    BufferAllocator allocator = new RootAllocator();
    IntVector vector1 = new IntVector("vector1", allocator);
    IntVector vector2 = new IntVector("vector1", allocator);
    IntVector vector3 = new IntVector(