Java Code Examples for org.apache.hadoop.io.retry.RetryProxy#create()
The following examples show how to use
org.apache.hadoop.io.retry.RetryProxy#create() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NativeS3FileSystem.java From RDFS with Apache License 2.0 | 6 votes |
private static NativeFileSystemStore createDefaultStore(Configuration conf) { NativeFileSystemStore store = new Jets3tNativeFileSystemStore(); RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( conf.getInt("fs.s3.maxRetries", 4), conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS); Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(IOException.class, basePolicy); exceptionToPolicyMap.put(S3Exception.class, basePolicy); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("storeFile", methodPolicy); return (NativeFileSystemStore) RetryProxy.create(NativeFileSystemStore.class, store, methodNameToPolicyMap); }
Example 2
Source File: S3FileSystem.java From hadoop-gpu with Apache License 2.0 | 6 votes |
private static FileSystemStore createDefaultStore(Configuration conf) { FileSystemStore store = new Jets3tFileSystemStore(); RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( conf.getInt("fs.s3.maxRetries", 4), conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS); Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(IOException.class, basePolicy); exceptionToPolicyMap.put(S3Exception.class, basePolicy); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>(); methodNameToPolicyMap.put("storeBlock", methodPolicy); methodNameToPolicyMap.put("retrieveBlock", methodPolicy); return (FileSystemStore) RetryProxy.create(FileSystemStore.class, store, methodNameToPolicyMap); }
Example 3
Source File: RMProxy.java From big-c with Apache License 2.0 | 6 votes |
/** * Create a proxy for the specified protocol. For non-HA, * this is a direct connection to the ResourceManager address. When HA is * enabled, the proxy handles the failover between the ResourceManagers as * well. */ @Private protected static <T> T createRMProxy(final Configuration configuration, final Class<T> protocol, RMProxy instance) throws IOException { YarnConfiguration conf = (configuration instanceof YarnConfiguration) ? (YarnConfiguration) configuration : new YarnConfiguration(configuration); RetryPolicy retryPolicy = createRetryPolicy(conf); if (HAUtil.isHAEnabled(conf)) { RMFailoverProxyProvider<T> provider = instance.createRMFailoverProxyProvider(conf, protocol); return (T) RetryProxy.create(protocol, provider, retryPolicy); } else { InetSocketAddress rmAddress = instance.getRMAddress(conf, protocol); LOG.info("Connecting to ResourceManager at " + rmAddress); T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress); return (T) RetryProxy.create(protocol, proxy, retryPolicy); } }
Example 4
Source File: S3FileSystem.java From big-c with Apache License 2.0 | 6 votes |
private static FileSystemStore createDefaultStore(Configuration conf) { FileSystemStore store = new Jets3tFileSystemStore(); RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( conf.getInt("fs.s3.maxRetries", 4), conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS); Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(IOException.class, basePolicy); exceptionToPolicyMap.put(S3Exception.class, basePolicy); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>(); methodNameToPolicyMap.put("storeBlock", methodPolicy); methodNameToPolicyMap.put("retrieveBlock", methodPolicy); return (FileSystemStore) RetryProxy.create(FileSystemStore.class, store, methodNameToPolicyMap); }
Example 5
Source File: NativeS3FileSystem.java From big-c with Apache License 2.0 | 6 votes |
private static NativeFileSystemStore createDefaultStore(Configuration conf) { NativeFileSystemStore store = new Jets3tNativeFileSystemStore(); RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( conf.getInt("fs.s3.maxRetries", 4), conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS); Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(IOException.class, basePolicy); exceptionToPolicyMap.put(S3Exception.class, basePolicy); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("storeFile", methodPolicy); methodNameToPolicyMap.put("rename", methodPolicy); return (NativeFileSystemStore) RetryProxy.create(NativeFileSystemStore.class, store, methodNameToPolicyMap); }
Example 6
Source File: NativeS3FileSystem.java From hadoop-gpu with Apache License 2.0 | 6 votes |
private static NativeFileSystemStore createDefaultStore(Configuration conf) { NativeFileSystemStore store = new Jets3tNativeFileSystemStore(); RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( conf.getInt("fs.s3.maxRetries", 4), conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS); Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(IOException.class, basePolicy); exceptionToPolicyMap.put(S3Exception.class, basePolicy); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); methodNameToPolicyMap.put("storeFile", methodPolicy); return (NativeFileSystemStore) RetryProxy.create(NativeFileSystemStore.class, store, methodNameToPolicyMap); }
Example 7
Source File: DFSClient.java From hadoop-gpu with Apache License 2.0 | 6 votes |
private static ClientProtocol createNamenode(ClientProtocol rpcNamenode) throws IOException { RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep( 5, LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS); Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class, createPolicy); Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>(); exceptionToPolicyMap.put(RemoteException.class, RetryPolicies.retryByRemoteException( RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap)); RetryPolicy methodPolicy = RetryPolicies.retryByException( RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap); Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>(); methodNameToPolicyMap.put("create", methodPolicy); return (ClientProtocol) RetryProxy.create(ClientProtocol.class, rpcNamenode, methodNameToPolicyMap); }
Example 8
Source File: RMProxy.java From hadoop with Apache License 2.0 | 6 votes |
/** * Create a proxy for the specified protocol. For non-HA, * this is a direct connection to the ResourceManager address. When HA is * enabled, the proxy handles the failover between the ResourceManagers as * well. */ @Private protected static <T> T createRMProxy(final Configuration configuration, final Class<T> protocol, RMProxy instance) throws IOException { YarnConfiguration conf = (configuration instanceof YarnConfiguration) ? (YarnConfiguration) configuration : new YarnConfiguration(configuration); RetryPolicy retryPolicy = createRetryPolicy(conf); if (HAUtil.isHAEnabled(conf)) { RMFailoverProxyProvider<T> provider = instance.createRMFailoverProxyProvider(conf, protocol); return (T) RetryProxy.create(protocol, provider, retryPolicy); } else { InetSocketAddress rmAddress = instance.getRMAddress(conf, protocol); LOG.info("Connecting to ResourceManager at " + rmAddress); T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress); return (T) RetryProxy.create(protocol, proxy, retryPolicy); } }
Example 9
Source File: NameNodeProxies.java From hadoop with Apache License 2.0 | 5 votes |
/** * Creates the namenode proxy with the passed protocol. This will handle * creation of either HA- or non-HA-enabled proxy objects, depending upon * if the provided URI is a configured logical URI. * * @param conf the configuration containing the required IPC * properties, client failover configurations, etc. * @param nameNodeUri the URI pointing either to a specific NameNode * or to a logical nameservice. * @param xface the IPC interface which should be created * @param fallbackToSimpleAuth set to true or false during calls to indicate if * a secure client falls back to simple auth * @return an object containing both the proxy and the associated * delegation token service it corresponds to * @throws IOException if there is an error creating the proxy **/ @SuppressWarnings("unchecked") public static <T> ProxyAndInfo<T> createProxy(Configuration conf, URI nameNodeUri, Class<T> xface, AtomicBoolean fallbackToSimpleAuth) throws IOException { AbstractNNFailoverProxyProvider<T> failoverProxyProvider = createFailoverProxyProvider(conf, nameNodeUri, xface, true, fallbackToSimpleAuth); if (failoverProxyProvider == null) { // Non-HA case return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface, UserGroupInformation.getCurrentUser(), true, fallbackToSimpleAuth); } else { // HA case Conf config = new Conf(conf); T proxy = (T) RetryProxy.create(xface, failoverProxyProvider, RetryPolicies.failoverOnNetworkException( RetryPolicies.TRY_ONCE_THEN_FAIL, config.maxFailoverAttempts, config.maxRetryAttempts, config.failoverSleepBaseMillis, config.failoverSleepMaxMillis)); Text dtService; if (failoverProxyProvider.useLogicalURI()) { dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri, HdfsConstants.HDFS_URI_SCHEME); } else { dtService = SecurityUtil.buildTokenService( NameNode.getAddress(nameNodeUri)); } return new ProxyAndInfo<T>(proxy, dtService, NameNode.getAddress(nameNodeUri)); } }
Example 10
Source File: TestRPC.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testWrappedStopProxy() throws IOException { StoppedProtocol wrappedProxy = RPC.getProxy(StoppedProtocol.class, StoppedProtocol.versionID, null, conf); StoppedInvocationHandler invocationHandler = (StoppedInvocationHandler) Proxy.getInvocationHandler(wrappedProxy); StoppedProtocol proxy = (StoppedProtocol) RetryProxy.create(StoppedProtocol.class, wrappedProxy, RetryPolicies.RETRY_FOREVER); assertEquals(0, invocationHandler.getCloseCalled()); RPC.stopProxy(proxy); assertEquals(1, invocationHandler.getCloseCalled()); }
Example 11
Source File: TestNodeStatusUpdater.java From hadoop with Apache License 2.0 | 5 votes |
@Override protected ResourceTracker getRMClient() throws IOException { RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf); resourceTracker = (ResourceTracker) RetryProxy.create(ResourceTracker.class, new MyResourceTracker6(rmStartIntervalMS, rmNeverStart), retryPolicy); return resourceTracker; }
Example 12
Source File: TestIPC.java From hadoop with Apache License 2.0 | 5 votes |
/** * Test the retry count while used in a retry proxy. */ @Test(timeout=60000) public void testRetryProxy() throws IOException { final Client client = new Client(LongWritable.class, conf); final TestServer server = new TestServer(1, false); server.callListener = new Runnable() { private int retryCount = 0; @Override public void run() { Assert.assertEquals(retryCount++, Server.getCallRetryCount()); } }; // try more times, so it is easier to find race condition bug // 10000 times runs about 6s on a core i7 machine final int totalRetry = 10000; DummyProtocol proxy = (DummyProtocol) Proxy.newProxyInstance( DummyProtocol.class.getClassLoader(), new Class[] { DummyProtocol.class }, new TestInvocationHandler(client, server, totalRetry)); DummyProtocol retryProxy = (DummyProtocol) RetryProxy.create( DummyProtocol.class, proxy, RetryPolicies.RETRY_FOREVER); try { server.start(); retryProxy.dummyRun(); Assert.assertEquals(TestInvocationHandler.retry, totalRetry + 1); } finally { Client.setCallIdAndRetryCount(0, 0); client.stop(); server.stop(); } }
Example 13
Source File: ServerProxy.java From big-c with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") protected static <T> T createRetriableProxy(final Configuration conf, final Class<T> protocol, final UserGroupInformation user, final YarnRPC rpc, final InetSocketAddress serverAddress, RetryPolicy retryPolicy) { T proxy = user.doAs(new PrivilegedAction<T>() { @Override public T run() { return (T) rpc.getProxy(protocol, serverAddress, conf); } }); return (T) RetryProxy.create(protocol, proxy, retryPolicy); }
Example 14
Source File: TestIPC.java From big-c with Apache License 2.0 | 5 votes |
/** * Test the retry count while used in a retry proxy. */ @Test(timeout=60000) public void testRetryProxy() throws IOException { final Client client = new Client(LongWritable.class, conf); final TestServer server = new TestServer(1, false); server.callListener = new Runnable() { private int retryCount = 0; @Override public void run() { Assert.assertEquals(retryCount++, Server.getCallRetryCount()); } }; // try more times, so it is easier to find race condition bug // 10000 times runs about 6s on a core i7 machine final int totalRetry = 10000; DummyProtocol proxy = (DummyProtocol) Proxy.newProxyInstance( DummyProtocol.class.getClassLoader(), new Class[] { DummyProtocol.class }, new TestInvocationHandler(client, server, totalRetry)); DummyProtocol retryProxy = (DummyProtocol) RetryProxy.create( DummyProtocol.class, proxy, RetryPolicies.RETRY_FOREVER); try { server.start(); retryProxy.dummyRun(); Assert.assertEquals(TestInvocationHandler.retry, totalRetry + 1); } finally { Client.setCallIdAndRetryCount(0, 0); client.stop(); server.stop(); } }
Example 15
Source File: NameNodeProxies.java From big-c with Apache License 2.0 | 5 votes |
/** * Creates the namenode proxy with the passed protocol. This will handle * creation of either HA- or non-HA-enabled proxy objects, depending upon * if the provided URI is a configured logical URI. * * @param conf the configuration containing the required IPC * properties, client failover configurations, etc. * @param nameNodeUri the URI pointing either to a specific NameNode * or to a logical nameservice. * @param xface the IPC interface which should be created * @param fallbackToSimpleAuth set to true or false during calls to indicate if * a secure client falls back to simple auth * @return an object containing both the proxy and the associated * delegation token service it corresponds to * @throws IOException if there is an error creating the proxy **/ @SuppressWarnings("unchecked") public static <T> ProxyAndInfo<T> createProxy(Configuration conf, URI nameNodeUri, Class<T> xface, AtomicBoolean fallbackToSimpleAuth) throws IOException { AbstractNNFailoverProxyProvider<T> failoverProxyProvider = createFailoverProxyProvider(conf, nameNodeUri, xface, true, fallbackToSimpleAuth); if (failoverProxyProvider == null) { // Non-HA case return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface, UserGroupInformation.getCurrentUser(), true, fallbackToSimpleAuth); } else { // HA case Conf config = new Conf(conf); T proxy = (T) RetryProxy.create(xface, failoverProxyProvider, RetryPolicies.failoverOnNetworkException( RetryPolicies.TRY_ONCE_THEN_FAIL, config.maxFailoverAttempts, config.maxRetryAttempts, config.failoverSleepBaseMillis, config.failoverSleepMaxMillis)); Text dtService; if (failoverProxyProvider.useLogicalURI()) { dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri, HdfsConstants.HDFS_URI_SCHEME); } else { dtService = SecurityUtil.buildTokenService( NameNode.getAddress(nameNodeUri)); } return new ProxyAndInfo<T>(proxy, dtService, NameNode.getAddress(nameNodeUri)); } }
Example 16
Source File: NameNodeProxies.java From big-c with Apache License 2.0 | 5 votes |
private static ClientProtocol createNNProxyWithClientProtocol( InetSocketAddress address, Configuration conf, UserGroupInformation ugi, boolean withRetries, AtomicBoolean fallbackToSimpleAuth) throws IOException { RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class); final RetryPolicy defaultPolicy = RetryUtils.getDefaultRetryPolicy( conf, DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT, DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY, DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT, SafeModeException.class); final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class); ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy( ClientNamenodeProtocolPB.class, version, address, ugi, conf, NetUtils.getDefaultSocketFactory(conf), org.apache.hadoop.ipc.Client.getTimeout(conf), defaultPolicy, fallbackToSimpleAuth).getProxy(); if (withRetries) { // create the proxy with retries Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); ClientProtocol translatorProxy = new ClientNamenodeProtocolTranslatorPB(proxy); return (ClientProtocol) RetryProxy.create( ClientProtocol.class, new DefaultFailoverProxyProvider<ClientProtocol>( ClientProtocol.class, translatorProxy), methodNameToPolicyMap, defaultPolicy); } else { return new ClientNamenodeProtocolTranslatorPB(proxy); } }
Example 17
Source File: TestNodeStatusUpdater.java From hadoop with Apache License 2.0 | 4 votes |
@Override protected ResourceTracker getRMClient() { RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf); return (ResourceTracker) RetryProxy.create(ResourceTracker.class, resourceTracker, retryPolicy); }
Example 18
Source File: TestNodeStatusUpdater.java From big-c with Apache License 2.0 | 4 votes |
@Override protected ResourceTracker getRMClient() { RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf); return (ResourceTracker) RetryProxy.create(ResourceTracker.class, resourceTracker, retryPolicy); }
Example 19
Source File: RMProxy.java From hadoop with Apache License 2.0 | 3 votes |
/** * @deprecated * This method is deprecated and is not used by YARN internally any more. * To create a proxy to the RM, use ClientRMProxy#createRMProxy or * ServerRMProxy#createRMProxy. * * Create a proxy to the ResourceManager at the specified address. * * @param conf Configuration to generate retry policy * @param protocol Protocol for the proxy * @param rmAddress Address of the ResourceManager * @param <T> Type information of the proxy * @return Proxy to the RM * @throws IOException */ @Deprecated public static <T> T createRMProxy(final Configuration conf, final Class<T> protocol, InetSocketAddress rmAddress) throws IOException { RetryPolicy retryPolicy = createRetryPolicy(conf); T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress); LOG.info("Connecting to ResourceManager at " + rmAddress); return (T) RetryProxy.create(protocol, proxy, retryPolicy); }
Example 20
Source File: RMProxy.java From big-c with Apache License 2.0 | 3 votes |
/** * @deprecated * This method is deprecated and is not used by YARN internally any more. * To create a proxy to the RM, use ClientRMProxy#createRMProxy or * ServerRMProxy#createRMProxy. * * Create a proxy to the ResourceManager at the specified address. * * @param conf Configuration to generate retry policy * @param protocol Protocol for the proxy * @param rmAddress Address of the ResourceManager * @param <T> Type information of the proxy * @return Proxy to the RM * @throws IOException */ @Deprecated public static <T> T createRMProxy(final Configuration conf, final Class<T> protocol, InetSocketAddress rmAddress) throws IOException { RetryPolicy retryPolicy = createRetryPolicy(conf); T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress); LOG.info("Connecting to ResourceManager at " + rmAddress); return (T) RetryProxy.create(protocol, proxy, retryPolicy); }