Skip to content

Commit 2005732

Browse files
skyflow-bhartiskyflow-bharti
committed
SK-3131 update the concurency limit and max data size limits
1 parent c57086d commit 2005732

5 files changed

Lines changed: 25 additions & 24 deletions

File tree

‎common/src/main/java/com/skyflow/errors/ErrorMessage.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,13 +90,13 @@ public enum ErrorMessage {
9090
InsufficientTokensPassedForTokenModeEnableStrict("%s0 Validation error. 'tokenMode' is set to 'ENABLE_STRICT', but some fields are missing tokens. Specify tokens for all fields."),
9191
BatchInsertPartialSuccess("%s0 Insert operation completed with partial success."),
9292
BatchInsertFailure("%s0 Insert operation failed."),
93-
RecordSizeExceedError("%s0 Maximum number of records exceeded. The limit is 10000."),
93+
RecordSizeExceedError("%s0 Maximum number of records exceeded. The limit is 100000."),
9494

9595
// Detokenize
9696
InvalidDetokenizeData("%s0 Validation error. Invalid detokenize data. Specify valid detokenize data."),
9797
EmptyDetokenizeData("%s0 Validation error. Invalid data tokens. Specify at least one data token."),
9898
EmptyTokenInDetokenizeData("%s0 Validation error. Invalid data tokens. Specify a valid data token."),
99-
TokensSizeExceedError("%s0 Maximum number of tokens exceeded. The limit is 10000."),
99+
TokensSizeExceedError("%s0 Maximum number of tokens exceeded. The limit is 100000."),
100100

101101
// Delete
102102
DeleteRequestNull("%s0 Validation error. DeleteRequest object is null. Specify a valid DeleteRequest object."),
@@ -105,7 +105,7 @@ public enum ErrorMessage {
105105
DeleteTokensRequestNull("%s0 Validation error. DeleteTokensRequest object is null. Specify a valid DeleteTokensRequest object."),
106106
EmptyDeleteTokensData("%s0 Validation error. Tokens list is empty. Specify at least one token to delete."),
107107
EmptyTokenInDeleteTokensData("%s0 Validation error. Invalid token in delete tokens request. Specify a valid token."),
108-
DeleteTokensSizeExceedError("%s0 Maximum number of tokens exceeded. The limit is 10000."),
108+
DeleteTokensSizeExceedError("%s0 Maximum number of tokens exceeded. The limit is 100000."),
109109

110110
// Get
111111
GetRequestNull("%s0 Validation error. GetRequest object is null. Specify a valid GetRequest object."),
@@ -165,7 +165,7 @@ public enum ErrorMessage {
165165
EmptyValueInTokenizeRecord("%s0 Validation error. Value in TokenizeRecord is null or empty. Specify a valid value."),
166166
EmptyTokenGroupNamesInTokenizeRecord("%s0 Validation error. TokenGroupNames in TokenizeRecord is null or empty. Specify at least one token group name."),
167167
EmptyTokenGroupNameInTokenizeRecord("%s0 Validation error. Token group name in TokenizeRecord is null or empty. Specify a valid token group name."),
168-
TokenizeDataSizeExceedError("%s0 Maximum number of tokenize records exceeded. The limit is 10000."),
168+
TokenizeDataSizeExceedError("%s0 Maximum number of tokenize records exceeded. The limit is 100000."),
169169
MissingIndexInBulkTokenizeRecord("%s0 Validation error. Index in BulkTokenizeRequestRecord is null. Specify an index for every record."),
170170
DuplicateIndexInBulkTokenizeRecord("%s0 Validation error. Duplicate index in BulkTokenizeRequestRecord. Specify a unique index for every record."),
171171

‎common/src/main/java/com/skyflow/logs/ErrorLogs.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,8 @@ public enum ErrorLogs {
6262
RECORDS_IS_REQUIRED("Invalid %s1 request. Records are required."),
6363
EMPTY_RECORDS("Invalid %s1 request. Records can not be empty."),
6464
INVALID_RECORD("Invalid %s1 request. Invalid record. Specify a valid record."),
65-
RECORD_SIZE_EXCEED("Maximum number of records exceeded. The limit is 10000."),
66-
TOKENS_SIZE_EXCEED("Maximum number of tokens exceeded. The limit is 10000."),
65+
RECORD_SIZE_EXCEED("Maximum number of records exceeded. The limit is 100000."),
66+
TOKENS_SIZE_EXCEED("Maximum number of tokens exceeded. The limit is 100000."),
6767
EMPTY_OR_NULL_VALUE_IN_VALUES("Invalid %s1 request. Value can not be null or empty in values for key \"%s2\"."),
6868
EMPTY_OR_NULL_KEY_IN_VALUES("Invalid %s1 request. Key can not be null or empty in values"),
6969
NULL_VALUE_ENTRY_IN_VALUES("Invalid %s1 request. Value entry in 'values' can not be null at index %s2."),
@@ -144,15 +144,15 @@ public enum ErrorLogs {
144144
EMPTY_VALUE_IN_TOKENIZE_RECORD("Invalid %s1 request. Value in TokenizeRecord can not be null or empty."),
145145
EMPTY_TOKEN_GROUP_NAMES_IN_TOKENIZE_RECORD("Invalid %s1 request. TokenGroupNames in TokenizeRecord can not be null or empty."),
146146
EMPTY_TOKEN_GROUP_NAME_IN_TOKENIZE_RECORD("Invalid %s1 request. Token group name in TokenizeRecord can not be null or empty at index %s2."),
147-
TOKENIZE_DATA_SIZE_EXCEED("Maximum number of tokenize records exceeded. The limit is 10000."),
147+
TOKENIZE_DATA_SIZE_EXCEED("Maximum number of tokenize records exceeded. The limit is 100000."),
148148
MISSING_INDEX_IN_BULK_TOKENIZE_RECORD("Invalid %s1 request. Index in BulkTokenizeRequestRecord can not be null at position %s2."),
149149
DUPLICATE_INDEX_IN_BULK_TOKENIZE_RECORD("Invalid %s1 request. Duplicate index %s2 in BulkTokenizeRequestRecord."),
150150
DELETE_REQUEST_NULL("Invalid %s1 request. Delete request can not be null."),
151151
DELETE_REQUEST_REJECTED("Delete request resulted in failure."),
152152
DELETE_TOKENS_REQUEST_NULL("Invalid %s1 request. DeleteTokens request can not be null."),
153153
EMPTY_DELETE_TOKENS_DATA("Invalid %s1 request. Delete tokens data can not be empty."),
154154
EMPTY_OR_NULL_TOKEN_IN_DELETE_TOKENS_DATA("Invalid %s1 request. Token can not be null or empty in delete tokens data at index %s2."),
155-
DELETE_TOKENS_SIZE_EXCEED("Maximum number of tokens exceeded. The limit is 10000."),
155+
DELETE_TOKENS_SIZE_EXCEED("Maximum number of tokens exceeded. The limit is 100000."),
156156
DELETE_TOKENS_REQUEST_REJECTED("DeleteTokens request resulted in failure."),
157157

158158
// invoke connection interface

‎flowvault/src/main/java/com/skyflow/VaultClient.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ protected void updateExecutorInHTTP() throws SkyflowException {
165165
// mode from this SDK is a SkyflowException, never a raw one.
166166
try {
167167
OkHttpClient.Builder httpBuilder = new OkHttpClient.Builder()
168-
.connectionPool(new ConnectionPool(10, 1, TimeUnit.MINUTES))
168+
.connectionPool(new ConnectionPool(100, 1, TimeUnit.MINUTES))
169169
// Overall ceiling; bounds the whole call including retries.
170170
.callTimeout(timeoutSeconds, TimeUnit.SECONDS)
171171
// OUTER: retries. Must wrap the auth interceptor so each attempt re-reads the

‎flowvault/src/main/java/com/skyflow/utils/Constants.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,23 +10,23 @@ public final class Constants extends BaseConstants {
1010
public static final String VAULT_DOMAIN = ".skyvault.";
1111
public static final String SDK_PREFIX;
1212
public static final String SDK_METRIC_NAME_VERSION_PREFIX = "skyflow-flowvault-java@";
13-
public static final Integer MAX_BULK_DATA_SIZE = 10000;
13+
public static final Integer MAX_BULK_DATA_SIZE = 100000;
1414
public static final Integer INSERT_BATCH_SIZE = 50;
1515
public static final Integer MAX_INSERT_BATCH_SIZE = 1000;
1616
public static final Integer INSERT_CONCURRENCY_LIMIT = 1;
17-
public static final Integer MAX_INSERT_CONCURRENCY_LIMIT = 10;
17+
public static final Integer MAX_INSERT_CONCURRENCY_LIMIT = 100;
1818
public static final Integer DETOKENIZE_BATCH_SIZE = 50;
1919
public static final Integer DETOKENIZE_CONCURRENCY_LIMIT = 1;
2020
public static final Integer MAX_DETOKENIZE_BATCH_SIZE = 1000;
21-
public static final Integer MAX_DETOKENIZE_CONCURRENCY_LIMIT = 10;
21+
public static final Integer MAX_DETOKENIZE_CONCURRENCY_LIMIT = 100;
2222
public static final Integer DELETE_TOKENS_BATCH_SIZE = 50;
2323
public static final Integer DELETE_TOKENS_CONCURRENCY_LIMIT = 1;
2424
public static final Integer MAX_DELETE_TOKENS_BATCH_SIZE = 1000;
25-
public static final Integer MAX_DELETE_TOKENS_CONCURRENCY_LIMIT = 10;
25+
public static final Integer MAX_DELETE_TOKENS_CONCURRENCY_LIMIT = 100;
2626
public static final Integer TOKENIZE_BATCH_SIZE = 50;
2727
public static final Integer TOKENIZE_CONCURRENCY_LIMIT = 1;
2828
public static final Integer MAX_TOKENIZE_BATCH_SIZE = 1000;
29-
public static final Integer MAX_TOKENIZE_CONCURRENCY_LIMIT = 10;
29+
public static final Integer MAX_TOKENIZE_CONCURRENCY_LIMIT = 100;
3030
public static final String DEFAULT_SDK_VERSION = "1.0.0";
3131
public static final String CONTEXT_KEY_REGEX = "^[a-zA-Z0-9_]+$";
3232

‎flowvault/src/test/java/com/skyflow/utils/validations/ValidationsTests.java‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import com.skyflow.enums.UpdateType;
1818
import com.skyflow.errors.ErrorMessage;
1919
import com.skyflow.errors.SkyflowException;
20+
import com.skyflow.utils.Constants;
2021
import com.skyflow.vault.data.BulkDeleteTokensRequest;
2122
import com.skyflow.vault.data.BulkDetokenizeRequest;
2223
import com.skyflow.vault.data.BulkInsertRequest;
@@ -988,13 +989,13 @@ public void testValidateBulkInsertRequest_validRequestWithUpsertAndTokens() {
988989
}
989990

990991
@Test
991-
public void testValidateBulkInsertRequest_over10000RecordsThrows() {
992+
public void testValidateBulkInsertRequest_overMaxBulkDataSizeRecordsThrows() {
992993
// Constants.MAX_BULK_DATA_SIZE is a hard ceiling; batching splits the payload but
993994
// does not lift it.
994995
Map<String, Object> data = new HashMap<>();
995996
data.put("name", "john");
996997
ArrayList<InsertRequestRecord> records = new ArrayList<>();
997-
for (int i = 0; i < 10001; i++) {
998+
for (int i = 0; i < Constants.MAX_BULK_DATA_SIZE + 1; i++) {
998999
records.add(BulkInsertRequestRecord.builder().tableName("table1").data(data).build());
9991000
}
10001001
BulkInsertRequest request = BulkInsertRequest.builder().records(records).build();
@@ -1007,11 +1008,11 @@ public void testValidateBulkInsertRequest_over10000RecordsThrows() {
10071008
}
10081009

10091010
@Test
1010-
public void testValidateBulkInsertRequest_exactly10000RecordsIsValid() {
1011+
public void testValidateBulkInsertRequest_exactlyMaxBulkDataSizeRecordsIsValid() {
10111012
Map<String, Object> data = new HashMap<>();
10121013
data.put("name", "john");
10131014
ArrayList<InsertRequestRecord> records = new ArrayList<>();
1014-
for (int i = 0; i < 10000; i++) {
1015+
for (int i = 0; i < Constants.MAX_BULK_DATA_SIZE; i++) {
10151016
records.add(BulkInsertRequestRecord.builder().tableName("table1").data(data).build());
10161017
}
10171018
BulkInsertRequest request = BulkInsertRequest.builder().records(records).build();
@@ -1023,9 +1024,9 @@ public void testValidateBulkInsertRequest_exactly10000RecordsIsValid() {
10231024
}
10241025

10251026
@Test
1026-
public void testValidateBulkDetokenizeRequest_over10000TokensThrows() {
1027+
public void testValidateBulkDetokenizeRequest_overMaxBulkDataSizeTokensThrows() {
10271028
List<String> tokens = new ArrayList<>();
1028-
for (int i = 0; i < 10001; i++) {
1029+
for (int i = 0; i < Constants.MAX_BULK_DATA_SIZE + 1; i++) {
10291030
tokens.add("token-" + i);
10301031
}
10311032
BulkDetokenizeRequest request = BulkDetokenizeRequest.builder().tokens(tokens).build();
@@ -1038,9 +1039,9 @@ public void testValidateBulkDetokenizeRequest_over10000TokensThrows() {
10381039
}
10391040

10401041
@Test
1041-
public void testValidateBulkDeleteTokensRequest_over10000TokensThrows() {
1042+
public void testValidateBulkDeleteTokensRequest_overMaxBulkDataSizeTokensThrows() {
10421043
List<String> tokens = new ArrayList<>();
1043-
for (int i = 0; i < 10001; i++) {
1044+
for (int i = 0; i < Constants.MAX_BULK_DATA_SIZE + 1; i++) {
10441045
tokens.add("token-" + i);
10451046
}
10461047
BulkDeleteTokensRequest request = BulkDeleteTokensRequest.builder().tokens(tokens).build();
@@ -1053,9 +1054,9 @@ public void testValidateBulkDeleteTokensRequest_over10000TokensThrows() {
10531054
}
10541055

10551056
@Test
1056-
public void testValidateBulkTokenizeRequest_over10000RecordsThrows() {
1057+
public void testValidateBulkTokenizeRequest_overMaxBulkDataSizeRecordsThrows() {
10571058
ArrayList<BulkTokenizeRequestRecord> records = new ArrayList<>();
1058-
for (int i = 0; i < 10001; i++) {
1059+
for (int i = 0; i < Constants.MAX_BULK_DATA_SIZE + 1; i++) {
10591060
records.add(BulkTokenizeRequestRecord.builder()
10601061
.value("value-" + i)
10611062
.tokenGroupNames(Collections.singletonList("group"))

0 commit comments

Comments
 (0)