Skip to content

Added support for add column and drop column #41

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open
6 changes: 6 additions & 0 deletions examples/scratch/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,12 @@ ice delete nyc.taxis --dry-run=false
# inspect
ice describe

# Alter table (add column)
ice alter-table flowers.iris --operations '[{"operation": "add column", "column_name": "age", "type": "int", "comment": "user age"}]'

# Alter table (drop column)
ice alter-table flowers.iris --operations '[{"operation": "drop column", "column_name": "age"}]'

# open ClickHouse shell, then try SQL below
clickhouse local
```
Expand Down
32 changes: 32 additions & 0 deletions ice/src/main/java/com/altinity/ice/cli/Main.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
package com.altinity.ice.cli;

import ch.qos.logback.classic.Level;
import com.altinity.ice.cli.internal.cmd.AlterTable;
import com.altinity.ice.cli.internal.cmd.Check;
import com.altinity.ice.cli.internal.cmd.CreateNamespace;
import com.altinity.ice.cli.internal.cmd.CreateTable;
Expand All @@ -34,6 +35,7 @@
import java.util.Base64;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.stream.Collectors;
import org.apache.iceberg.catalog.Namespace;
Expand Down Expand Up @@ -428,6 +430,36 @@ void deleteNamespace(
}
}

@CommandLine.Command(
name = "alter-table",
description =
"Alter table schema by adding or dropping columns. Supported operations: add column, drop column. Example: ice alter-table ns1.table1 --operations '[{\"operation\": \"add column\", \"column_name\": \"age\", \"type\": \"int\", \"comment\": \"user age\"}]'")
void alterTable(
@CommandLine.Parameters(
arity = "1",
paramLabel = "<name>",
description = "Table name (e.g. ns1.table1)")
String name,
@CommandLine.Option(
names = {"--operations"},
required = true,
description =
"JSON array of operations: [{'operation': 'add column', 'column_name': 'age', 'type': 'int', 'comment': 'user age'}, {'operation': 'drop column', 'column_name': 'col_name'}]")
String operationsJson)
throws IOException {
try (RESTCatalog catalog = loadCatalog(this.configFile())) {
TableIdentifier tableId = TableIdentifier.parse(name);

ObjectMapper mapper = newObjectMapper();
List<Map<String, String>> operations =
mapper.readValue(
operationsJson,
mapper.getTypeFactory().constructCollectionType(List.class, Map.class));

AlterTable.run(catalog, tableId, operations);
}
}

@CommandLine.Command(name = "delete", description = "Delete data from catalog.")
void delete(
@CommandLine.Parameters(
Expand Down
189 changes: 189 additions & 0 deletions ice/src/main/java/com/altinity/ice/cli/internal/cmd/AlterTable.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,189 @@
/*
* Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*/
package com.altinity.ice.cli.internal.cmd;

import com.fasterxml.jackson.annotation.JsonProperty;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import org.apache.iceberg.Table;
import org.apache.iceberg.Transaction;
import org.apache.iceberg.UpdateSchema;
import org.apache.iceberg.catalog.Catalog;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.types.Types;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class AlterTable {
private static final Logger logger = LoggerFactory.getLogger(AlterTable.class);

// Static constants for column definition keys
private static final String OPERATION_KEY = "operation";
private static final String COLUMN_NAME_KEY = "column_name";
private static final String TYPE_KEY = "type";
private static final String COMMENT_KEY = "comment";

private AlterTable() {}

public enum OperationType {
ADD("add column"),
DROP("drop column");

private final String key;

OperationType(String key) {
this.key = key;
}

public String getKey() {
return key;
}

public static OperationType fromKey(String key) {
for (OperationType type : values()) {
if (type.key.equals(key)) {
return type;
}
}
throw new IllegalArgumentException("Unsupported operation type: " + key);
}
}

public record ColumnDefinition(
@JsonProperty("operation") String operation,
@JsonProperty("column_name") String columnName,
@JsonProperty("type") String type,
@JsonProperty("comment") String comment) {}

public static void run(
Catalog catalog, TableIdentifier tableId, List<Map<String, String>> operations)
throws IOException {

Table table = catalog.loadTable(tableId);

// Apply schema changes
Transaction transaction = table.newTransaction();
UpdateSchema updateSchema = transaction.updateSchema();

for (Map<String, String> operation : operations) {

// get the operation type
ColumnDefinition columnDef = parseColumnDefinitionMap(operation);
OperationType operationType = OperationType.fromKey(columnDef.operation());

switch (operationType) {
case ADD:
Types.NestedField field =
parseColumnType(columnDef.columnName(), columnDef.type(), columnDef.comment());
updateSchema.addColumn(columnDef.columnName(), field.type(), columnDef.comment());
logger.info("Adding column '{}' to table: {}", columnDef.columnName(), tableId);
break;
case DROP:
// Validate that the column exists
if (table.schema().findField(columnDef.columnName()) == null) {
throw new IllegalArgumentException(
"Column '" + columnDef.columnName() + "' does not exist in table: " + tableId);
}
updateSchema.deleteColumn(columnDef.columnName());
logger.info("Dropping column '{}' from table: {}", columnDef.columnName(), tableId);
break;
default:
throw new IllegalArgumentException("Unsupported operation type: " + operationType);
}
}

updateSchema.commit();
transaction.commitTransaction();

logger.info("Successfully applied {} operations to table: {}", operations.size(), tableId);
}

static ColumnDefinition parseColumnDefinitionMap(Map<String, String> operationMap) {
try {
String operation = operationMap.get(OPERATION_KEY);
String columnName = operationMap.get(COLUMN_NAME_KEY);
String type = operationMap.get(TYPE_KEY);
String comment = operationMap.get(COMMENT_KEY);

if (operation == null || operation.trim().isEmpty()) {
throw new IllegalArgumentException(OPERATION_KEY + " is required and cannot be empty");
}

if (columnName == null || columnName.trim().isEmpty()) {
throw new IllegalArgumentException(COLUMN_NAME_KEY + " is required and cannot be empty");
}

// For drop column operations, type is not required
OperationType operationType = OperationType.fromKey(operation);
if (operationType == OperationType.ADD) {
if (type == null || type.trim().isEmpty()) {
throw new IllegalArgumentException(
TYPE_KEY + " is required and cannot be empty for add column operations");
}
if (comment == null || comment.trim().isEmpty()) {
throw new IllegalArgumentException(
COMMENT_KEY + " is required and cannot be empty for add column operations");
}
}

return new ColumnDefinition(operation, columnName, type, comment);
} catch (Exception e) {
throw new IllegalArgumentException("Invalid column definition: " + e.getMessage());
}
}

private static Types.NestedField parseColumnType(String name, String type, String comment) {
Types.NestedField field;

switch (type.toLowerCase()) {
case "string":
case "varchar":
field = Types.NestedField.optional(-1, name, Types.StringType.get(), comment);
break;
case "int":
case "integer":
field = Types.NestedField.optional(-1, name, Types.IntegerType.get(), comment);
break;
case "long":
case "bigint":
field = Types.NestedField.optional(-1, name, Types.LongType.get(), comment);
break;
case "double":
field = Types.NestedField.optional(-1, name, Types.DoubleType.get(), comment);
break;
case "float":
field = Types.NestedField.optional(-1, name, Types.FloatType.get(), comment);
break;
case "boolean":
field = Types.NestedField.optional(-1, name, Types.BooleanType.get(), comment);
break;
case "date":
field = Types.NestedField.optional(-1, name, Types.DateType.get(), comment);
break;
case "timestamp":
field = Types.NestedField.optional(-1, name, Types.TimestampType.withoutZone(), comment);
break;
case "timestamptz":
field = Types.NestedField.optional(-1, name, Types.TimestampType.withZone(), comment);
break;
case "binary":
field = Types.NestedField.optional(-1, name, Types.BinaryType.get(), comment);
break;
default:
throw new IllegalArgumentException(
"Unsupported column type: "
+ type
+ ". Supported types: string, int, long, double, float, boolean, date, timestamp, timestamptz, binary");
}

return field;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
/*
* Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*/
package com.altinity.ice.cli.internal.cmd;

import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNull;

import java.util.HashMap;
import java.util.Map;
import org.testng.annotations.Test;

public class AlterTableTest {

@Test
public void testParseColumnDefinitionJson() {

Map<String, String> operation = new HashMap<>();
operation.put("operation", "add column");
operation.put("column_name", "age");
operation.put("type", "int");
operation.put("comment", "User age");
AlterTable.ColumnDefinition columnDef = AlterTable.parseColumnDefinitionMap(operation);

assertEquals(columnDef.columnName(), "age");
assertEquals(columnDef.type(), "int");
assertEquals(columnDef.comment(), "User age");
}

@Test(expectedExceptions = IllegalArgumentException.class)
public void testParseColumnDefinitionJsonWithoutComment() {

Map<String, String> operation = new HashMap<>();
operation.put("operation", "add column");
operation.put("column_name", "name");
operation.put("type", "string");
AlterTable.ColumnDefinition columnDef = AlterTable.parseColumnDefinitionMap(operation);

assertEquals(columnDef.columnName(), "name");
assertEquals(columnDef.type(), "string");
assertNull(columnDef.comment());
}

@Test(expectedExceptions = IllegalArgumentException.class)
public void testParseColumnDefinitionJsonMissingColumnName() {

Map<String, String> operation = new HashMap<>();
operation.put("operation", "add column");
operation.put("type", "int");
operation.put("comment", "User age");
AlterTable.parseColumnDefinitionMap(operation);
}

@Test(expectedExceptions = IllegalArgumentException.class)
public void testParseColumnDefinitionJsonMissingType() {

Map<String, String> operation = new HashMap<>();
operation.put("operation", "add column");
operation.put("column_name", "age");
operation.put("comment", "User age");
AlterTable.parseColumnDefinitionMap(operation);
}

@Test(expectedExceptions = IllegalArgumentException.class)
public void testParseColumnDefinitionJsonEmptyColumnName() {

Map<String, String> operation = new HashMap<>();
operation.put("operation", "add column");
operation.put("column_name", "");
operation.put("type", "int");
AlterTable.parseColumnDefinitionMap(operation);
}

@Test(expectedExceptions = IllegalArgumentException.class)
public void testParseColumnDefinitionJsonEmptyType() {

Map<String, String> operation = new HashMap<>();
operation.put("operation", "add column");
operation.put("column_name", "age");
operation.put("type", "");
AlterTable.parseColumnDefinitionMap(operation);
}

@Test(expectedExceptions = IllegalArgumentException.class)
public void testParseColumnDefinitionJsonInvalidJson() {

Map<String, String> operation = new HashMap<>();
operation.put("operation2", "add column");
operation.put("column_name", "age");
operation.put("type", "int");
AlterTable.parseColumnDefinitionMap(operation);
}

@Test
public void testOperationTypeFromKey() {
assertEquals(AlterTable.OperationType.fromKey("add column"), AlterTable.OperationType.ADD);
assertEquals(AlterTable.OperationType.fromKey("drop column"), AlterTable.OperationType.DROP);
}

@Test(expectedExceptions = IllegalArgumentException.class)
public void testOperationTypeFromKeyInvalid() {
AlterTable.OperationType.fromKey("invalid");
}

@Test
public void testParseColumnDefinitionMapForDropColumn() {

// Test drop column operation - only needs column_name
Map<String, String> operation = new HashMap<>();
operation.put("operation", "drop column");
operation.put("column_name", "age");
AlterTable.ColumnDefinition columnDef = AlterTable.parseColumnDefinitionMap(operation);

assertEquals(columnDef.operation(), "drop column");
assertEquals(columnDef.columnName(), "age");
assertNull(columnDef.type());
assertNull(columnDef.comment());
}
}