我正在遍历一个dynamodb表并建立一个包含所有表行的csv文件。但是,csv中的大多数条目都是重复的。dynamodb表中大约有350k条唯一记录(所有记录都是唯一的)。生成的文件(使用下面的代码)只有大约4k个唯一的条目,其余的都是重复的。
public void fetchItems() throws IOException {
AmazonDynamoDB amazonDynamoDB = DynamoDBClient.getInstance().getConnection();
ScanResult result = null;
ScanRequest req = new ScanRequest().withTableName("TABLE_NAME");
List<Map<String, AttributeValue>> records = new ArrayList<Map<String, AttributeValue>>();
do {
if (result != null) {
req.setExclusiveStartKey(result.getLastEvaluatedKey());
}
result = amazonDynamoDB.scan(req);
List<Map<String, AttributeValue>> rows = result.getItems();
Map<String, AttributeValue> staticColumnRecord = new HashMap<String, AttributeValue>();
for (Map<String, AttributeValue> map : rows) {
List<String> snowFlakeColumnsListApparel = new ArrayList<String>(Arrays.asList(apparelColumns));
for (String key : snowFlakeColumnsListApparel) {
AttributeValue value = map.get(key);
if (value != null) {
staticColumnRecord.put(key, value);
} else {
staticColumnRecord.put(key, new AttributeValue().withS("null"));
}
}
records.add(staticColumnRecord);
}
} while (result.getLastEvaluatedKey() != null);
buildCSV(records);
}
private void buildCSV(List<Map<String, AttributeValue>> changedRecords) throws IOException {
List<String> headers = changedRecords.stream().flatMap(map -> map.keySet().stream()).distinct()
.collect(Collectors.toList());
try (FileOutputStream fos = new FileOutputStream(file);
OutputStreamWriter bwr = new OutputStreamWriter(fos, StandardCharsets.UTF_8)) {
StringBuffer headerContent = new StringBuffer();
for (String string : headers) {
headerContent.append(string);
headerContent.append(",");
}
StringBuffer strBuilder = new StringBuffer();
for (Map<String, AttributeValue> lmap : changedRecords) {
StringBuilder stringBuilder = new StringBuilder("");
String sep = "";
for (Entry<String, AttributeValue> string2 : lmap.entrySet()) {
String value = string2.getValue().getS();
stringBuilder.append(sep).append("\"").append(value).append("\"");
sep = ",";
}
if (!stringBuilder.toString().isEmpty()) {
strBuilder.append(stringBuilder).append(System.getProperty("line.separator"));
}
}
headerContent.append("\n");
headerContent.append(strBuilder);
bwr.write(headerContent.toString());
} catch (IOException e) {
e.printStackTrace();
}
}
暂无答案!
目前还没有任何答案,快来回答吧!