Skip to content

Commit 28e48ae

Browse files
shangm2Presto CUDF CI
authored andcommitted
Clean up legacy MetaDataUpdater and ConnectorTypeSerde (prestodb#25652)
## Description 1. Extract the cleanup from prestodb#25242 ## Motivation and Context 1. clean up the codebase 2. separate concern from the original pr ## Impact <!---Describe any public API or user-facing feature change or any performance impact--> ## Test Plan 1. passed verifier test 2. ## Contributor checklist - [ ] Please make sure your submission complies with our [contributing guide](https://github.com/prestodb/presto/blob/master/CONTRIBUTING.md), in particular [code style](https://github.com/prestodb/presto/blob/master/CONTRIBUTING.md#code-style) and [commit standards](https://github.com/prestodb/presto/blob/master/CONTRIBUTING.md#commit-standards). - [ ] PR description addresses the issue accurately and concisely. If the change is non-trivial, a GitHub Issue is referenced. - [ ] Documented new properties (with its default value), SQL syntax, functions, or other functionality. - [ ] If release notes are required, they follow the [release notes guidelines](https://github.com/prestodb/presto/wiki/Release-Notes-Guidelines). - [ ] Adequate tests were added if applicable. - [ ] CI passed. ## Release Notes ``` == NO RELEASE NOTE == ```
1 parent 32fe958 commit 28e48ae

File tree

108 files changed

+228
-3317
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

108 files changed

+228
-3317
lines changed

presto-hive-hadoop2/src/test/java/com/facebook/presto/hive/s3select/S3SelectTestHelper.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
import com.facebook.presto.hive.HiveColumnHandle;
3131
import com.facebook.presto.hive.HiveCommonClientConfig;
3232
import com.facebook.presto.hive.HiveEncryptionInformationProvider;
33-
import com.facebook.presto.hive.HiveFileRenamer;
3433
import com.facebook.presto.hive.HiveHdfsConfiguration;
3534
import com.facebook.presto.hive.HiveLocationService;
3635
import com.facebook.presto.hive.HiveMetadataFactory;
@@ -178,7 +177,6 @@ public S3SelectTestHelper(String host,
178177
new HivePartitionObjectBuilder(),
179178
new HiveEncryptionInformationProvider(ImmutableSet.of()),
180179
new HivePartitionStats(),
181-
new HiveFileRenamer(),
182180
columnConverterProvider,
183181
new QuickStatsProvider(metastoreClient, hdfsEnvironment, DO_NOTHING_DIRECTORY_LISTER, new HiveClientConfig(), new NamenodeStats(), ImmutableList.of()),
184182
new HiveTableWritabilityChecker(config));

presto-hive/pom.xml

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -160,11 +160,6 @@
160160
<scope>test</scope>
161161
</dependency>
162162

163-
<dependency>
164-
<groupId>com.facebook.drift</groupId>
165-
<artifactId>drift-api</artifactId>
166-
</dependency>
167-
168163
<dependency>
169164
<groupId>com.google.guava</groupId>
170165
<artifactId>guava</artifactId>
@@ -267,11 +262,6 @@
267262
<artifactId>jackson-databind</artifactId>
268263
</dependency>
269264

270-
<dependency>
271-
<groupId>javax.annotation</groupId>
272-
<artifactId>javax.annotation-api</artifactId>
273-
</dependency>
274-
275265
<dependency>
276266
<groupId>org.roaringbitmap</groupId>
277267
<artifactId>RoaringBitmap</artifactId>

presto-hive/src/main/java/com/facebook/presto/hive/ForUpdatingHiveMetadata.java

Lines changed: 0 additions & 31 deletions
This file was deleted.

presto-hive/src/main/java/com/facebook/presto/hive/HiveClientConfig.java

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -169,8 +169,6 @@ public class HiveClientConfig
169169
private DataSize pageFileStripeMaxSize = new DataSize(24, MEGABYTE);
170170
private boolean parquetDereferencePushdownEnabled;
171171

172-
private int maxMetadataUpdaterThreads = 100;
173-
174172
private boolean isPartialAggregationPushdownEnabled;
175173
private boolean isPartialAggregationPushdownForVariableLengthDatatypesEnabled;
176174

@@ -683,6 +681,7 @@ public HiveClientConfig setMaxPartitionsPerWriter(int maxPartitionsPerWriter)
683681
this.maxPartitionsPerWriter = maxPartitionsPerWriter;
684682
return this;
685683
}
684+
686685
public int getWriteValidationThreads()
687686
{
688687
return writeValidationThreads;
@@ -1372,19 +1371,6 @@ public boolean isParquetDereferencePushdownEnabled()
13721371
return this.parquetDereferencePushdownEnabled;
13731372
}
13741373

1375-
@Min(1)
1376-
public int getMaxMetadataUpdaterThreads()
1377-
{
1378-
return maxMetadataUpdaterThreads;
1379-
}
1380-
1381-
@Config("hive.max-metadata-updater-threads")
1382-
public HiveClientConfig setMaxMetadataUpdaterThreads(int maxMetadataUpdaterThreads)
1383-
{
1384-
this.maxMetadataUpdaterThreads = maxMetadataUpdaterThreads;
1385-
return this;
1386-
}
1387-
13881374
@Config("hive.partial_aggregation_pushdown_enabled")
13891375
@ConfigDescription("enable partial aggregation pushdown")
13901376
public HiveClientConfig setPartialAggregationPushdownEnabled(boolean partialAggregationPushdownEnabled)

presto-hive/src/main/java/com/facebook/presto/hive/HiveClientModule.java

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -69,13 +69,11 @@
6969
import com.facebook.presto.parquet.cache.ParquetCacheConfig;
7070
import com.facebook.presto.parquet.cache.ParquetFileMetadata;
7171
import com.facebook.presto.parquet.cache.ParquetMetadataSource;
72-
import com.facebook.presto.spi.connector.ConnectorMetadataUpdaterProvider;
7372
import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider;
7473
import com.facebook.presto.spi.connector.ConnectorPageSinkProvider;
7574
import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
7675
import com.facebook.presto.spi.connector.ConnectorPlanOptimizerProvider;
7776
import com.facebook.presto.spi.connector.ConnectorSplitManager;
78-
import com.facebook.presto.spi.connector.ConnectorTypeSerdeProvider;
7977
import com.google.common.cache.Cache;
8078
import com.google.common.cache.CacheBuilder;
8179
import com.google.common.collect.ImmutableList;
@@ -98,7 +96,6 @@
9896
import static com.facebook.airlift.configuration.ConfigBinder.configBinder;
9997
import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder;
10098
import static com.facebook.airlift.json.smile.SmileCodecBinder.smileCodecBinder;
101-
import static com.facebook.drift.codec.guice.ThriftCodecBinder.thriftCodecBinder;
10299
import static com.facebook.presto.orc.StripeMetadataSource.CacheableRowGroupIndices;
103100
import static com.facebook.presto.orc.StripeMetadataSource.CacheableSlice;
104101
import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator;
@@ -152,9 +149,6 @@ public void configure(Binder binder)
152149
binder.bind(HiveWriterStats.class).in(Scopes.SINGLETON);
153150
newExporter(binder).export(HiveWriterStats.class).as(generatedNameOf(HiveWriterStats.class, connectorId));
154151

155-
binder.bind(HiveFileRenamer.class).in(Scopes.SINGLETON);
156-
newExporter(binder).export(HiveFileRenamer.class).as(generatedNameOf(HiveFileRenamer.class, connectorId));
157-
158152
newSetBinder(binder, EventClient.class).addBinding().to(HiveEventClient.class).in(Scopes.SINGLETON);
159153
binder.bind(HivePartitionManager.class).in(Scopes.SINGLETON);
160154
newExporter(binder).export(HivePartitionManager.class).withGeneratedName();
@@ -177,11 +171,8 @@ public void configure(Binder binder)
177171
binder.bind(ConnectorPageSinkProvider.class).to(HivePageSinkProvider.class).in(Scopes.SINGLETON);
178172
binder.bind(ConnectorNodePartitioningProvider.class).to(HiveNodePartitioningProvider.class).in(Scopes.SINGLETON);
179173
binder.bind(ConnectorPlanOptimizerProvider.class).to(HivePlanOptimizerProvider.class).in(Scopes.SINGLETON);
180-
binder.bind(ConnectorMetadataUpdaterProvider.class).to(HiveMetadataUpdaterProvider.class).in(Scopes.SINGLETON);
181-
binder.bind(ConnectorTypeSerdeProvider.class).to(HiveConnectorTypeSerdeProvider.class).in(Scopes.SINGLETON);
182174
binder.install(new ThriftCodecModule());
183175
binder.install(new DefaultThriftCodecsModule());
184-
thriftCodecBinder(binder).bindThriftCodec(HiveMetadataUpdateHandle.class);
185176

186177
jsonCodecBinder(binder).bindJsonCodec(PartitionUpdate.class);
187178
smileCodecBinder(binder).bindSmileCodec(PartitionUpdate.class);
@@ -261,14 +252,6 @@ public ExecutorService createCachingHiveMetastoreExecutor(HiveConnectorId hiveCl
261252
daemonThreadsNamed("hive-metastore-" + hiveClientId + "-%s"));
262253
}
263254

264-
@ForUpdatingHiveMetadata
265-
@Singleton
266-
@Provides
267-
public ExecutorService createUpdatingHiveMetadataExecutor(HiveConnectorId hiveClientId)
268-
{
269-
return newCachedThreadPool(daemonThreadsNamed("hive-metadata-updater-" + hiveClientId + "-%s"));
270-
}
271-
272255
@ForFileRename
273256
@Singleton
274257
@Provides

presto-hive/src/main/java/com/facebook/presto/hive/HiveConnector.java

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,12 @@
2222
import com.facebook.presto.spi.connector.ConnectorCapabilities;
2323
import com.facebook.presto.spi.connector.ConnectorCommitHandle;
2424
import com.facebook.presto.spi.connector.ConnectorMetadata;
25-
import com.facebook.presto.spi.connector.ConnectorMetadataUpdaterProvider;
2625
import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider;
2726
import com.facebook.presto.spi.connector.ConnectorPageSinkProvider;
2827
import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
2928
import com.facebook.presto.spi.connector.ConnectorPlanOptimizerProvider;
3029
import com.facebook.presto.spi.connector.ConnectorSplitManager;
3130
import com.facebook.presto.spi.connector.ConnectorTransactionHandle;
32-
import com.facebook.presto.spi.connector.ConnectorTypeSerdeProvider;
3331
import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorMetadata;
3432
import com.facebook.presto.spi.procedure.Procedure;
3533
import com.facebook.presto.spi.session.PropertyMetadata;
@@ -73,8 +71,6 @@ public class HiveConnector
7371
private final ConnectorAccessControl accessControl;
7472
private final ClassLoader classLoader;
7573
private final ConnectorPlanOptimizerProvider planOptimizerProvider;
76-
private final ConnectorMetadataUpdaterProvider metadataUpdaterProvider;
77-
private final ConnectorTypeSerdeProvider connectorTypeSerdeProvider;
7874

7975
private final HiveTransactionManager transactionManager;
8076

@@ -94,8 +90,6 @@ public HiveConnector(
9490
List<PropertyMetadata<?>> analyzeProperties,
9591
ConnectorAccessControl accessControl,
9692
ConnectorPlanOptimizerProvider planOptimizerProvider,
97-
ConnectorMetadataUpdaterProvider metadataUpdaterProvider,
98-
ConnectorTypeSerdeProvider connectorTypeSerdeProvider,
9993
ClassLoader classLoader)
10094
{
10195
this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null");
@@ -114,8 +108,6 @@ public HiveConnector(
114108
this.accessControl = requireNonNull(accessControl, "accessControl is null");
115109
this.classLoader = requireNonNull(classLoader, "classLoader is null");
116110
this.planOptimizerProvider = requireNonNull(planOptimizerProvider, "planOptimizerProvider is null");
117-
this.metadataUpdaterProvider = requireNonNull(metadataUpdaterProvider, "metadataUpdaterProvider is null");
118-
this.connectorTypeSerdeProvider = requireNonNull(connectorTypeSerdeProvider, "connectorTypeSerdeProvider is null");
119111
}
120112

121113
@Override
@@ -156,18 +148,6 @@ public ConnectorPlanOptimizerProvider getConnectorPlanOptimizerProvider()
156148
return planOptimizerProvider;
157149
}
158150

159-
@Override
160-
public ConnectorMetadataUpdaterProvider getConnectorMetadataUpdaterProvider()
161-
{
162-
return metadataUpdaterProvider;
163-
}
164-
165-
@Override
166-
public ConnectorTypeSerdeProvider getConnectorTypeSerdeProvider()
167-
{
168-
return connectorTypeSerdeProvider;
169-
}
170-
171151
@Override
172152
public Set<SystemTable> getSystemTables()
173153
{

presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorFactory.java

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,11 @@
3838
import com.facebook.presto.spi.connector.ConnectorAccessControl;
3939
import com.facebook.presto.spi.connector.ConnectorContext;
4040
import com.facebook.presto.spi.connector.ConnectorFactory;
41-
import com.facebook.presto.spi.connector.ConnectorMetadataUpdaterProvider;
4241
import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider;
4342
import com.facebook.presto.spi.connector.ConnectorPageSinkProvider;
4443
import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
4544
import com.facebook.presto.spi.connector.ConnectorPlanOptimizerProvider;
4645
import com.facebook.presto.spi.connector.ConnectorSplitManager;
47-
import com.facebook.presto.spi.connector.ConnectorTypeSerdeProvider;
4846
import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorPageSinkProvider;
4947
import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorPageSourceProvider;
5048
import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorSplitManager;
@@ -158,8 +156,6 @@ public Connector create(String catalogName, Map<String, String> config, Connecto
158156
ConnectorAccessControl accessControl = new SystemTableAwareAccessControl(injector.getInstance(ConnectorAccessControl.class));
159157
Set<Procedure> procedures = injector.getInstance(Key.get(new TypeLiteral<Set<Procedure>>() {}));
160158
ConnectorPlanOptimizerProvider planOptimizerProvider = injector.getInstance(ConnectorPlanOptimizerProvider.class);
161-
ConnectorMetadataUpdaterProvider metadataUpdaterProvider = injector.getInstance(ConnectorMetadataUpdaterProvider.class);
162-
ConnectorTypeSerdeProvider connectorTypeSerdeProvider = injector.getInstance(ConnectorTypeSerdeProvider.class);
163159

164160
List<PropertyMetadata<?>> allSessionProperties = new ArrayList<>(hiveSessionProperties.getSessionProperties());
165161
allSessionProperties.addAll(hiveCommonSessionProperties.getSessionProperties());
@@ -180,8 +176,6 @@ public Connector create(String catalogName, Map<String, String> config, Connecto
180176
hiveAnalyzeProperties.getAnalyzeProperties(),
181177
accessControl,
182178
planOptimizerProvider,
183-
metadataUpdaterProvider,
184-
connectorTypeSerdeProvider,
185179
classLoader);
186180
}
187181
catch (Exception e) {

presto-hive/src/main/java/com/facebook/presto/hive/HiveConnectorTypeSerdeProvider.java

Lines changed: 0 additions & 48 deletions
This file was deleted.

0 commit comments

Comments
 (0)