com.ikanow.aleph2.data_import_manager.harvest.modules.LocalHarvestTestModule.java Source code

Java tutorial

Introduction

Here is the source code for com.ikanow.aleph2.data_import_manager.harvest.modules.LocalHarvestTestModule.java

Source

/*******************************************************************************
 * Copyright 2015, The IKANOW Open Source Project.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *******************************************************************************/
package com.ikanow.aleph2.data_import_manager.harvest.modules;

import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.Optional;
import java.util.Scanner;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.ikanow.aleph2.data_import.services.HarvestContext;
import com.ikanow.aleph2.core.shared.utils.ClassloaderUtils;
import com.ikanow.aleph2.data_model.interfaces.data_import.IHarvestContext;
import com.ikanow.aleph2.data_model.interfaces.data_import.IHarvestTechnologyModule;
import com.ikanow.aleph2.data_model.interfaces.data_services.IManagementDbService;
import com.ikanow.aleph2.data_model.interfaces.shared_services.ICrudService;
import com.ikanow.aleph2.data_model.interfaces.shared_services.IServiceContext;
import com.ikanow.aleph2.data_model.objects.data_import.DataBucketBean;
import com.ikanow.aleph2.data_model.objects.shared.BasicMessageBean;
import com.ikanow.aleph2.data_model.objects.shared.GlobalPropertiesBean;
import com.ikanow.aleph2.data_model.utils.BeanTemplateUtils;
import com.ikanow.aleph2.data_model.utils.CrudUtils;
import com.ikanow.aleph2.data_model.utils.JsonUtils;
import com.ikanow.aleph2.data_model.utils.CrudUtils.SingleQueryComponent;
import com.ikanow.aleph2.data_model.utils.ErrorUtils;
import com.ikanow.aleph2.data_model.utils.ModuleUtils;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;

import fj.data.Either;
import fj.data.Validation;

/** A test module for developing harvesters (not maintained - was mainly written for early development and 
 *  at some point either needs to be made useful for standalone testing by 3rd party devs or removed)
 *  THIS CLASS HAS NO COVERAGE SO NEED TO HANDLE TEST ON MODIFICATION
 * @author Alex
 */
public class LocalHarvestTestModule {

    protected final IServiceContext _context;
    protected final IManagementDbService _core_management_db;
    protected final IManagementDbService _underlying_management_db;
    protected final GlobalPropertiesBean _globals;

    protected @Inject Injector _injector;

    /** Guice constructor, injects underlying management service (must actually be mongodb)
     * @param management_db_service
     */
    @Inject
    public LocalHarvestTestModule(IServiceContext service_context) {
        _context = service_context;
        _core_management_db = _context.getCoreManagementDbService();
        _underlying_management_db = _context.getService(IManagementDbService.class, Optional.empty()).get();
        _globals = service_context.getGlobalProperties();
    }

    public void start(String source_key, String harvest_tech_jar_path, String... commands) throws Exception {
        if ((null == commands) || (0 == commands.length)) {
            try (Scanner scanner = new Scanner(System.in)) {
                while (true) {
                    System.out.println("Enter command (or help/exit): ");
                    String command = scanner.nextLine();
                    if (command.equals("exit")) {
                        System.exit(0);
                    }
                    run_command(source_key, harvest_tech_jar_path, command);
                }
            }
        } else
            for (String command : commands) {
                run_command(source_key, harvest_tech_jar_path, command);
            }
    }

    /** Entry point
     * @param args - config_file source_key harvest_tech_id
     * @throws Exception 
     */
    public static void main(final String[] args) {
        try {
            if (args.length < 3) {
                System.out.println("CLI: config_file source_key harvest_tech_jar_path");
                System.exit(-1);
            }
            System.out.println("Running with command line: " + Arrays.toString(args));
            Config config = ConfigFactory.parseFile(new File(args[0]));

            LocalHarvestTestModule app = ModuleUtils.initializeApplication(Arrays.asList(), Optional.of(config),
                    Either.left(LocalHarvestTestModule.class));
            app.start(args[1], args[2], Arrays.copyOfRange(args, 3, args.length));
        } catch (Exception e) {
            try {
                System.out.println("Got all the way to main");
                e.printStackTrace();
            } catch (Exception e2) { // the exception failed!
                System.out.println(ErrorUtils.getLongForm("Got all the way to main: {0}", e));
            }
        }
    }

    /** Actually perform harvester command
     * @param source_key
     * @param harvest_tech_jar_path
     * @param command
     * @throws ExecutionException 
     * @throws InterruptedException 
     * @throws IOException 
     * @throws JsonMappingException 
     * @throws JsonParseException 
     */
    private void run_command(String source_key, String harvest_tech_jar_path, String command) throws Exception {

        @SuppressWarnings("unchecked")
        final ICrudService<JsonNode> v1_config_db = _underlying_management_db
                .getUnderlyingPlatformDriver(ICrudService.class, Optional.of("ingest.source")).get();

        final SingleQueryComponent<JsonNode> query = CrudUtils.allOf().when("key", source_key);
        final Optional<JsonNode> result = v1_config_db.getObjectBySpec(query).get();

        if (!result.isPresent()) {
            System.out.println("Must specify valid source.key: " + source_key);
            return;
        }

        // Create a bucket out of the source

        DataBucketBean bucket = createBucketFromSource(result.get());

        // OK now we simply create an instance of the harvester and invoke it

        final Validation<BasicMessageBean, IHarvestTechnologyModule> ret_val = ClassloaderUtils
                .getFromCustomClasspath(IHarvestTechnologyModule.class,
                        "com.ikanow.aleph2.test.example.ExampleHarvestTechnology",
                        Optional.of(new File(harvest_tech_jar_path).getAbsoluteFile().toURI().toString()),
                        Collections.emptyList(), "test1", command);

        final IHarvestContext context = _injector.getInstance(HarvestContext.class);

        if (ret_val.isFail()) {
            System.out.println("Failed to instantiate harvester: " + ret_val.fail().message());
        } else {
            final IHarvestTechnologyModule harvester = ret_val.success();
            if (command.equals("canRunOnThisNode")) {
                System.out.println(command + ": " + harvester.canRunOnThisNode(bucket, context));
            } else {
                CompletableFuture<BasicMessageBean> harvest_result = null; // (this is most of them)
                if (command.equals("onDelete")) {
                    harvest_result = harvester.onDelete(bucket, context);
                } else if (command.equals("onHarvestComplete")) {
                    harvest_result = harvester.onHarvestComplete(bucket, context);
                } else if (command.equals("onNewSource")) {
                    harvest_result = harvester.onNewSource(bucket, context, true);
                } else if (command.equals("onPeriodicPoll")) {
                    harvest_result = harvester.onPeriodicPoll(bucket, context);
                } else if (command.equals("onPurge")) {
                    harvest_result = harvester.onPurge(bucket, context);
                } else if (command.equals("onTestSource")) {
                    harvest_result = harvester.onTestSource(bucket, null, context);
                } else if (command.equals("onUpdatedSource")) {
                    harvest_result = harvester.onUpdatedSource(bucket, bucket, true, Optional.empty(), context);
                } else {
                    if (command.equals("help")) {
                        System.out.println("Allowed commands: ");
                    } else {
                        System.out.println("Command not recognized, allowed commands: ");
                    }
                    System.out.println(Arrays.asList(harvester.getClass().getMethods()).stream()
                            .map(m -> m.getName()).collect(Collectors.joining(",")));
                }
                if (null != harvest_result) {
                    System.out.println(command + ": success: " + harvest_result.get().success());
                    System.out.println(command + ": source: " + harvest_result.get().source());
                    System.out.println(command + ": message: " + harvest_result.get().message());
                } else {
                    System.out.println("(no return value)");
                }
            }
        }
    }

    private static DataBucketBean createBucketFromSource(JsonNode src_json) throws Exception {

        @SuppressWarnings("unused")
        final String _id = safeJsonGet(JsonUtils._ID, src_json).asText(); // (think we'll use key instead of _id?)
        final String key = safeJsonGet("key", src_json).asText();
        final String created = safeJsonGet("created", src_json).asText();
        final String modified = safeJsonGet("modified", src_json).asText();
        final String title = safeJsonGet("title", src_json).asText();
        final String description = safeJsonGet("description", src_json).asText();
        final String owner_id = safeJsonGet("ownerId", src_json).asText();

        final JsonNode tags = safeJsonGet("tags", src_json); // collection of strings
        //TODO (ALEPH-19): need to convert the DB authentication across to the Aleph2 format 
        @SuppressWarnings("unused")
        final JsonNode comm_ids = safeJsonGet("communityIds", src_json); // collection of strings
        final JsonNode px_pipeline = safeJsonGet("processingPipeline", src_json); // collection of JSON objects, first one should have data_bucket
        final JsonNode px_pipeline_first_el = px_pipeline.get(0);
        final JsonNode data_bucket = safeJsonGet("data_bucket", px_pipeline_first_el);

        final DataBucketBean bucket = BeanTemplateUtils.build(data_bucket, DataBucketBean.class)
                .with(DataBucketBean::_id, key).with(DataBucketBean::created, parseJavaDate(created))
                .with(DataBucketBean::modified, parseJavaDate(modified)).with(DataBucketBean::display_name, title)
                .with(DataBucketBean::description, description).with(DataBucketBean::owner_id, owner_id)
                .with(DataBucketBean::tags, StreamSupport.stream(tags.spliterator(), false).map(jt -> jt.asText())
                        .collect(Collectors.toSet()))
                .done().get();

        return bucket;
    }

    private static Date parseJavaDate(String java_date_tostring_format) throws ParseException {
        return new SimpleDateFormat("EEE MMM d HH:mm:ss zzz yyyy").parse(java_date_tostring_format);
    }

    private static JsonNode safeJsonGet(String fieldname, JsonNode src) {
        final JsonNode j = Optional.ofNullable(src.get(fieldname)).orElse(JsonNodeFactory.instance.objectNode());
        //DEBUG
        //System.out.println(j);
        return j;
    }
}