xbird.xquery.expr.ext.BDQExpr.java Source code

Java tutorial

Introduction

Here is the source code for xbird.xquery.expr.ext.BDQExpr.java

Source

/*
 * @(#)$Id$
 *
 * Copyright 2006-2008 Makoto YUI
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * Contributors:
 *     Makoto YUI - initial implementation
 */
package xbird.xquery.expr.ext;

import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
import java.rmi.RemoteException;
import java.rmi.server.UnicastRemoteObject;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import xbird.engine.XQEngineClient;
import xbird.engine.Request.ReturnType;
import xbird.engine.remote.RemoteSequence;
import xbird.engine.remote.RemoteSequenceProxy;
import xbird.engine.request.CompileRequest;
import xbird.engine.request.PreparedQueryRequest;
import xbird.engine.request.QueryRequest;
import xbird.util.concurrent.ExecutorFactory;
import xbird.util.concurrent.ExecutorUtils;
import xbird.util.net.TimeoutSocketProdiver;
import xbird.xquery.DynamicError;
import xbird.xquery.XQueryException;
import xbird.xquery.dm.value.Item;
import xbird.xquery.dm.value.Sequence;
import xbird.xquery.dm.value.SingleItem;
import xbird.xquery.dm.value.sequence.ChainedSequence;
import xbird.xquery.dm.value.sequence.MarshalledSequence;
import xbird.xquery.dm.value.sequence.PipelinedSequence;
import xbird.xquery.dm.value.sequence.ValueSequence;
import xbird.xquery.expr.AbstractXQExpression;
import xbird.xquery.expr.LiteralExpr;
import xbird.xquery.expr.XQExpression;
import xbird.xquery.expr.flwr.ForClause;
import xbird.xquery.expr.flwr.LetClause;
import xbird.xquery.expr.opt.ShippedVariable;
import xbird.xquery.expr.opt.ThreadedVariable;
import xbird.xquery.expr.path.axis.AxisStep;
import xbird.xquery.expr.var.BindingVariable;
import xbird.xquery.expr.var.VarRef;
import xbird.xquery.expr.var.Variable;
import xbird.xquery.expr.var.BindingVariable.ExecHostVariable;
import xbird.xquery.expr.var.BindingVariable.ForVariable;
import xbird.xquery.expr.var.BindingVariable.LetVariable;
import xbird.xquery.func.ext.RemoteEval;
import xbird.xquery.meta.DynamicContext;
import xbird.xquery.meta.IFocus;
import xbird.xquery.meta.StaticContext;
import xbird.xquery.meta.XQueryContext;
import xbird.xquery.parser.XQueryParserVisitor;
import xbird.xquery.parser.visitor.AbstractXQueryParserVisitor;

/**
 * BDQExpr ::= "execute at" [ VarRef "in" ] Expr "{" Expr "}"
 * <DIV lang="en"></DIV>
 * <DIV lang="ja"></DIV>
 * 
 * @author Makoto YUI (yuin405+xbird@gmail.com)
 */
public final class BDQExpr extends AbstractXQExpression implements Externalizable {
    private static final long serialVersionUID = 2302133457373935360L;
    private static final Log LOG = LogFactory.getLog(BDQExpr.class);

    private XQExpression _endpoint;
    private BindingVariable _hostVar; // may be null
    private XQExpression _queryExpr;

    private boolean _parallel = false;

    public BDQExpr(XQExpression endpoint, BindingVariable hostVar, XQExpression expr) throws XQueryException {
        super();
        if (endpoint == null) {
            throw new IllegalArgumentException();
        }
        if (expr == null) {
            throw new IllegalArgumentException();
        }
        this._endpoint = endpoint;
        this._hostVar = hostVar;
        this._queryExpr = expr;
        this._type = expr.getType();
    }

    public BDQExpr() {
    } // for Externalizable

    public boolean isParallel() {
        return _parallel;
    }

    public void setParallel(boolean parallel) {
        this._parallel = parallel;
    }

    public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
        this._endpoint = (XQExpression) in.readObject();
        this._hostVar = (ExecHostVariable) in.readObject();
        final XQExpression queryExpr = (XQExpression) in.readObject();
        this._queryExpr = queryExpr;
        this._type = queryExpr.getType();
    }

    public void writeExternal(ObjectOutput out) throws IOException {
        out.writeObject(_endpoint);
        out.writeObject(_hostVar);
        out.writeObject(_queryExpr);
    }

    public XQExpression getRemoteEndpoint() {
        return _endpoint;
    }

    public XQExpression getBodyExpression() {
        return _queryExpr;
    }

    public void setBodyExpression(XQExpression body) {
        this._queryExpr = body;
    }

    public XQExpression visit(XQueryParserVisitor visitor, XQueryContext ctxt) throws XQueryException {
        return visitor.visit(this, ctxt);
    }

    public XQExpression staticAnalysis(StaticContext statEnv) throws XQueryException {
        if (!_analyzed) {
            this._analyzed = true;

            XQExpression analyzedEndpoint = _endpoint.staticAnalysis(statEnv);
            this._endpoint = analyzedEndpoint;

            // prepare to ship local variables
            final List<ShippedVariable> shippedVars = new ArrayList<ShippedVariable>(4);
            ParametricVariableDetector detector = new ParametricVariableDetector(shippedVars, statEnv);
            detector.visit(_queryExpr, null);

            final XQExpression compiledExpr;
            if (analyzedEndpoint instanceof LiteralExpr) {// distributed compile                 
                LiteralExpr epLiteral = (LiteralExpr) analyzedEndpoint;
                String endpoint = epLiteral.getValue().stringValue();

                final XQEngineClient client = new XQEngineClient(endpoint);
                final CompileRequest request = new CompileRequest(_queryExpr);

                if (LOG.isInfoEnabled()) {
                    LOG.info("Invoking remote compilation at [" + endpoint + "]:\n " + _queryExpr);
                }

                // invokes remote compilation
                final Object result;
                try {
                    result = client.execute(request);
                } catch (RemoteException e) {
                    throw new XQueryException(e.getMessage(), e.getCause());
                } finally {
                    try {
                        client.close();
                    } catch (RemoteException e) {
                        LOG.warn("shutdown failed for `" + endpoint + '\'', e);
                    }
                }

                compiledExpr = (XQExpression) result;
                // reset local expression
                if (!shippedVars.isEmpty()) {
                    ShippedVariableModifier modifier = new ShippedVariableModifier(shippedVars);
                    modifier.visit(compiledExpr, null);
                }

                this._parallel = false;

            } else { // compile locally (some access paths such as index access is disabled) 
                boolean prevState = statEnv.isIndicesAccessible();
                statEnv.setIndicesAccessible(false); //TODO remote compilation for static endpoint
                compiledExpr = _queryExpr.staticAnalysis(statEnv);
                statEnv.setIndicesAccessible(prevState);
            }

            this._queryExpr = compiledExpr;
            this._type = compiledExpr.getType();
            final ThreadedVariable threadedVar = new ThreadedVariable(this);
            statEnv.addThreadedVariable(threadedVar);
            return threadedVar;
        }
        return this;
    }

    private static final class ParametricVariableDetector extends AbstractXQueryParserVisitor {
        private final List<ShippedVariable> _variablesHolder;
        private final StaticContext _statEnv;
        private int _counter = 0;

        ParametricVariableDetector(List<ShippedVariable> variablesHolder, StaticContext statEnv) {
            super();
            this._variablesHolder = variablesHolder;
            this._statEnv = statEnv;
        }

        @Override
        public ForClause visit(ForClause clause, XQueryContext ctxt) throws XQueryException {
            ForVariable var = clause.getVariable();
            var.setInsideRemoteExpr(true);
            XQExpression e = clause.getExpression();
            e.visit(this, ctxt);
            return clause;
        }

        @Override
        public LetClause visit(LetClause clause, XQueryContext ctxt) throws XQueryException {
            LetVariable lv = clause.getVariable();
            lv.setInsideRemoteExpr(true);
            XQExpression e = clause.getExpression();
            e.visit(this, ctxt);
            return clause;
        }

        @Override
        public XQExpression visit(VarRef ref, XQueryContext ctxt) throws XQueryException {
            Variable var = ref.getValue();
            if (!var.isInsideRemoteExpr()) {
                if (var.isAnalyzed()) {
                    var.staticAnalysis(_statEnv);
                    ShippedVariable shippedVar = new ShippedVariable(var, _counter++);
                    _variablesHolder.add(shippedVar);
                    ref.setValue(shippedVar);
                }
            }
            return ref;
        }
    }

    private static final class ShippedVariableModifier extends AbstractXQueryParserVisitor {
        private final List<ShippedVariable> _variablesHolder;

        ShippedVariableModifier(List<ShippedVariable> variablesHolder) {
            super();
            this._variablesHolder = variablesHolder;
        }

        @Override
        public XQExpression visit(VarRef ref, XQueryContext ctxt) throws XQueryException {
            final Variable var = ref.getValue();
            if (var instanceof ShippedVariable) {
                ShippedVariable shippedVar = (ShippedVariable) var;
                int shippedId = shippedVar.getIdentifier();
                ShippedVariable original = _variablesHolder.get(shippedId);
                XQExpression originalExpr = original.getValue();
                shippedVar.setValue(originalExpr);
                shippedVar.setLocal(true);
            }
            return ref;
        }
    }

    private static final class OutsideNonDownwardAxisDetector extends AbstractXQueryParserVisitor {

        private boolean foundBDQExpr = false;
        private boolean foundNonDownwardAxis = false;

        public OutsideNonDownwardAxisDetector() {
            super();
        }

        @Override
        public XQExpression visit(BDQExpr expr, XQueryContext ctxt) throws XQueryException {
            foundBDQExpr = true;
            return expr;
        }

        @Override
        public XQExpression visit(AxisStep step, XQueryContext ctxt) throws XQueryException {
            if (foundBDQExpr && step.isNonDownwardAxis()) {
                this.foundNonDownwardAxis = true;
            }
            return step;
        }

        public boolean foundNonDownwardAxis() {
            return foundNonDownwardAxis;
        }

    }

    private static final class InsideNonDownwardAxisDetector extends AbstractXQueryParserVisitor {

        private boolean foundShippedVariable = false;
        private boolean foundNonDownwardAxis = false;

        public InsideNonDownwardAxisDetector() {
            super();
        }

        @Override
        public XQExpression visit(BDQExpr expr, XQueryContext ctxt) throws XQueryException {
            expr.getBodyExpression().visit(this, ctxt);
            return expr;
        }

        @Override
        public XQExpression visit(Variable variable, XQueryContext ctxt) throws XQueryException {
            if (variable instanceof ShippedVariable) {
                this.foundShippedVariable = true;
            }
            return super.visit(variable, ctxt);
        }

        @Override
        public XQExpression visit(AxisStep step, XQueryContext ctxt) throws XQueryException {
            if (foundShippedVariable && step.isNonDownwardAxis()) {
                this.foundNonDownwardAxis = true;
            }
            return step;
        }

        public boolean foundNonDownwardAxis() {
            return foundNonDownwardAxis;
        }

    }

    public Sequence<? extends Item> eval(Sequence<? extends Item> contextSeq, DynamicContext dynEnv)
            throws XQueryException {
        XQExpression outsideQueryExpr = dynEnv.getQueryExpression();
        OutsideNonDownwardAxisDetector outsideNdaDetector = new OutsideNonDownwardAxisDetector();
        outsideNdaDetector.visit(outsideQueryExpr, null);
        final PreparedQueryRequest request = new PreparedQueryRequest(_queryExpr,
                outsideNdaDetector.foundNonDownwardAxis() ? ReturnType.REMOTE_PADED_SEQUENCE
                        : RemoteEval.RETURN_TYPE);

        InsideNonDownwardAxisDetector insideNdaDetector = new InsideNonDownwardAxisDetector();
        insideNdaDetector.visit(_queryExpr, null);
        PreparedQueryRequest svRequest = insideNdaDetector.foundNonDownwardAxis()
                ? new PreparedQueryRequest(_queryExpr, ReturnType.REMOTE_PADED_SEQUENCE)
                : request;

        final ArrayList<ShippedVariable> shippedVars = new ArrayList<ShippedVariable>(4);
        ShippedVariableCollector collector = new ShippedVariableCollector(shippedVars);
        collector.visit(_queryExpr, null);
        final int numVars = shippedVars.size();
        for (int i = 0; i < numVars; i++) {
            ShippedVariable sv = shippedVars.get(i);
            prepareVariablesToShip(svRequest, sv, contextSeq, dynEnv);
        }

        final List<String> endpoints = getEndpoints(_endpoint, contextSeq, dynEnv);
        final int numEndpoints = endpoints.size();
        if (numEndpoints == 0) {
            return ValueSequence.EMPTY_SEQUENCE;
        }
        final Sequence result;
        if (numEndpoints == 1) {
            result = invokeRequest(endpoints.get(0), request);
        } else if (_parallel) {
            result = invokeRequestsInParallel(endpoints, _hostVar, request, dynEnv);
        } else {
            Sequence lastSeq = invokeRequest(endpoints.get(0), request);
            for (int i = 1; i < numEndpoints; i++) {
                Sequence secSeq = invokeRequest(endpoints.get(i), request);
                lastSeq = new ChainedSequence(lastSeq, secSeq, dynEnv);
            }
            result = lastSeq;
        }
        return result;
    }

    private static Sequence invokeRequestsInParallel(final List<String> endpoints, final BindingVariable hostVar,
            final PreparedQueryRequest request, final DynamicContext dynEnv) {
        final int numEndpoints = endpoints.size();
        final ExecutorService ex = ExecutorFactory.newFixedThreadPool(numEndpoints, "parallelRemoteExec");
        final Sequence[] results = new Sequence[numEndpoints];
        //final AtomicReferenceArray<Sequence> resultsRef = new AtomicReferenceArray<Sequence>(results);
        try {
            for (int i = 0; i < numEndpoints; i++) {
                final String endpoint = endpoints.get(i);
                final int at = i;
                Runnable r = new Runnable() {
                    public void run() {
                        final Sequence res;
                        try {
                            res = invokeRequest(endpoint, request);
                        } catch (XQueryException e) {
                            throw new IllegalStateException("An error caused while evaluating CompiledQueryRequest#"
                                    + request.getIdentifier() + " at " + endpoint, e);
                        }
                        results[at] = res;
                    }
                };
                ex.execute(r);
            }
        } finally {
            ExecutorUtils.shutdownAndAwaitTermination(ex);
        }
        assert (ex.isTerminated());
        return new PipelinedSequence(dynEnv, results);
    }

    private static Sequence invokeRequest(final String endpoint, final PreparedQueryRequest request)
            throws XQueryException {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Invoking remote execution at [" + endpoint + "]:\n " + request.getCompiledExpression());
        }
        final XQEngineClient client = new XQEngineClient(endpoint);
        final Object result;
        try {
            result = client.execute(request);
        } catch (RemoteException e) {
            throw new XQueryException(e.getMessage(), e.getCause());
        } finally {
            try {
                client.close();
            } catch (RemoteException e) {
                LOG.warn("shutdown failed for `" + endpoint + '\'', e);
            }
        }
        Sequence resultSeq = (Sequence) result;
        return resultSeq;
    }

    private static final class ShippedVariableCollector extends AbstractXQueryParserVisitor {
        private final List<ShippedVariable> _variablesHolder;

        public ShippedVariableCollector(final List<ShippedVariable> holder) {
            super();
            this._variablesHolder = holder;
        }

        @Override
        public XQExpression visit(VarRef ref, XQueryContext ctxt) throws XQueryException {
            final Variable var = ref.getValue();
            if (var instanceof ShippedVariable) {
                _variablesHolder.add((ShippedVariable) var);
            }
            return ref;
        }
    }

    private static List<String> getEndpoints(final XQExpression endpointExpr,
            final Sequence<? extends Item> contextSeq, final DynamicContext dynEnv) throws XQueryException {
        final Sequence ep = endpointExpr.eval(contextSeq, dynEnv);
        final IFocus<? extends Item> epFocus = ep.iterator();
        if (!epFocus.hasNext()) {
            epFocus.closeQuietly();
            throw new DynamicError("Invalid XQueryD expression. Endpoint does not found");
        }
        final List<String> endpoints = new ArrayList<String>(4);
        do {
            Item firstItem = epFocus.next();
            String endpointStr = firstItem.stringValue();
            endpoints.add(endpointStr);
        } while (epFocus.hasNext());
        epFocus.closeQuietly();
        return endpoints;
    }

    private static void prepareVariablesToShip(final QueryRequest request, final ShippedVariable shippedVar,
            final Sequence<? extends Item> contextSeq, final DynamicContext dynEnv) throws XQueryException {
        final Sequence result = shippedVar.eval(contextSeq, dynEnv);
        final ReturnType rettype = request.getReturnType();
        if (RemoteEval.ENV_NOWRAP_VARSHIP || !rettype.isRemoteSequnece()) {
            shippedVar.setResult(new MarshalledSequence(result, dynEnv));
        } else if (result instanceof SingleItem) {
            shippedVar.setResult(result);
        } else {
            final RemoteSequenceProxy proxy = new RemoteSequenceProxy(result, request);
            try {
                UnicastRemoteObject.exportObject(proxy, 0, TimeoutSocketProdiver.createClientSocketFactory(),
                        TimeoutSocketProdiver.createServerSocketFactory());
            } catch (RemoteException e) {
                throw new XQueryException("failed exporting variable: " + shippedVar.getName(), e);
            }
            final RemoteSequence remote = new RemoteSequence(proxy, result.getType());
            shippedVar.setResult(remote);
        }
    }
}