Example usage for org.apache.hadoop.mapred ShuffleHandler loadVersion

Introduction

In this page you can find the example usage for org.apache.hadoop.mapred ShuffleHandler loadVersion.

Prototype

@VisibleForTesting
    Version loadVersion() throws IOException 

Source Link

Usage

From source file:org.apache.tez.auxservices.TestShuffleHandler.java

License:Apache License

@Test
public void testRecoveryFromOtherVersions() throws IOException {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(12345, 1);
    final File tmpDir = new File(System.getProperty("test.build.data", System.getProperty("java.io.tmpdir")),
            TestShuffleHandler.class.getName());
    Configuration conf = new Configuration();
    conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
    conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
    ShuffleHandler shuffle = new ShuffleHandler();
    // emulate aux services startup with recovery enabled
    shuffle.setRecoveryPath(new Path(tmpDir.toString()));
    tmpDir.mkdirs();/*w w w. j a  v  a  2 s.com*/
    try {
        shuffle.init(conf);
        shuffle.start();

        // setup a shuffle token for an application
        DataOutputBuffer outputBuffer = new DataOutputBuffer();
        outputBuffer.reset();
        Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>("identifier".getBytes(),
                "password".getBytes(), new Text(user), new Text("shuffleService"));
        jt.write(outputBuffer);
        shuffle.initializeApplication(new ApplicationInitializationContext(user, appId,
                ByteBuffer.wrap(outputBuffer.getData(), 0, outputBuffer.getLength())));

        // verify we are authorized to shuffle
        int rc = getShuffleResponseCode(shuffle, jt);
        Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);

        // emulate shuffle handler restart
        shuffle.close();
        shuffle = new ShuffleHandler();
        shuffle.setRecoveryPath(new Path(tmpDir.toString()));
        shuffle.init(conf);
        shuffle.start();

        // verify we are still authorized to shuffle to the old application
        rc = getShuffleResponseCode(shuffle, jt);
        Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);
        Version version = Version.newInstance(1, 0);
        Assert.assertEquals(version, shuffle.getCurrentVersion());

        // emulate shuffle handler restart with compatible version
        Version version11 = Version.newInstance(1, 1);
        // update version info before close shuffle
        shuffle.storeVersion(version11);
        Assert.assertEquals(version11, shuffle.loadVersion());
        shuffle.close();
        shuffle = new ShuffleHandler();
        shuffle.setRecoveryPath(new Path(tmpDir.toString()));
        shuffle.init(conf);
        shuffle.start();
        // shuffle version will be override by CURRENT_VERSION_INFO after restart
        // successfully.
        Assert.assertEquals(version, shuffle.loadVersion());
        // verify we are still authorized to shuffle to the old application
        rc = getShuffleResponseCode(shuffle, jt);
        Assert.assertEquals(HttpURLConnection.HTTP_OK, rc);

        // emulate shuffle handler restart with incompatible version
        Version version21 = Version.newInstance(2, 1);
        shuffle.storeVersion(version21);
        Assert.assertEquals(version21, shuffle.loadVersion());
        shuffle.close();
        shuffle = new ShuffleHandler();
        shuffle.setRecoveryPath(new Path(tmpDir.toString()));
        shuffle.init(conf);

        try {
            shuffle.start();
            Assert.fail("Incompatible version, should expect fail here.");
        } catch (ServiceStateException e) {
            Assert.assertTrue("Exception message mismatch",
                    e.getMessage().contains("Incompatible version for state DB schema:"));
        }

    } finally {
        if (shuffle != null) {
            shuffle.close();
        }
        FileUtil.fullyDelete(tmpDir);
    }
}

---