com.qubole.presto.kinesis.KinesisTableDescriptionSupplier.java Source code

Introduction

Here is the source code for com.qubole.presto.kinesis.KinesisTableDescriptionSupplier.java

Source

/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.qubole.presto.kinesis;

import com.qubole.presto.kinesis.s3config.S3TableConfigClient;
import io.airlift.json.JsonCodec;
import io.airlift.log.Logger;

import java.nio.file.DirectoryIteratorException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.Path;

import java.io.IOException;
import java.util.List;
import java.util.Map;

import org.weakref.jmx.internal.guava.base.Objects;

import com.facebook.presto.spi.SchemaTableName;
import java.util.function.Supplier;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Inject;

import static java.util.Objects.requireNonNull;

/**
 *
 * This class get() method reads the table description file stored in Kinesis directory
 * and then creates user defined field for Presto Table.
 *
 */
public class KinesisTableDescriptionSupplier
        implements Supplier<Map<SchemaTableName, KinesisStreamDescription>>, ConnectorShutdown {
    private static final Logger log = Logger.get(KinesisTableDescriptionSupplier.class);

    private final KinesisConnectorConfig kinesisConnectorConfig;
    private final JsonCodec<KinesisStreamDescription> streamDescriptionCodec;
    private final S3TableConfigClient s3TableConfigClient;

    @Inject
    KinesisTableDescriptionSupplier(KinesisConnectorConfig kinesisConnectorConfig,
            JsonCodec<KinesisStreamDescription> streamDescriptionCodec, S3TableConfigClient anS3Client) {
        this.kinesisConnectorConfig = requireNonNull(kinesisConnectorConfig, "kinesisConnectorConfig is null");
        this.streamDescriptionCodec = requireNonNull(streamDescriptionCodec, "streamDescriptionCodec is null");
        this.s3TableConfigClient = requireNonNull(anS3Client, "S3 table config client is null");
    }

    @Override
    public Map<SchemaTableName, KinesisStreamDescription> get() {
        if (this.s3TableConfigClient.isUsingS3()) {
            return this.s3TableConfigClient.getTablesFromS3();
        } else {
            return getTablesFromDirectory();
        }
    }

    public Map<SchemaTableName, KinesisStreamDescription> getTablesFromDirectory() {
        ImmutableMap.Builder<SchemaTableName, KinesisStreamDescription> builder = ImmutableMap.builder();
        try {
            for (Path file : listFiles(Paths.get(kinesisConnectorConfig.getTableDescriptionDir()))) {
                if (Files.isRegularFile(file) && file.getFileName().toString().endsWith("json")) {
                    KinesisStreamDescription table = streamDescriptionCodec.fromJson(Files.readAllBytes(file));
                    String schemaName = Objects.firstNonNull(table.getSchemaName(),
                            kinesisConnectorConfig.getDefaultSchema());
                    log.debug("Kinesis table %s %s %s", schemaName, table.getTableName(), table);
                    builder.put(new SchemaTableName(schemaName, table.getTableName()), table);
                }
            }

            Map<SchemaTableName, KinesisStreamDescription> tableDefinitions = builder.build();
            log.debug("Loaded table definitions: %s", tableDefinitions.keySet());

            return tableDefinitions;
        } catch (IOException e) {
            log.warn(e, "Error: ");
            throw Throwables.propagate(e);
        }
    }

    /** Shutdown any periodic update jobs. */
    @Override
    public void shutdown() {
        this.s3TableConfigClient.shutdown();
        return;
    }

    private static List<Path> listFiles(Path dir) {
        if ((dir != null) && Files.isDirectory(dir)) {
            try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
                ImmutableList.Builder<Path> builder = ImmutableList.builder();
                for (Path file : stream) {
                    builder.add(file);
                }

                return builder.build();
            } catch (IOException | DirectoryIteratorException x) {
                log.warn(x, "Warning.");
                throw Throwables.propagate(x);
            }
        }
        return ImmutableList.of();
    }
}