fix: Address edge cases for excluding large properties when using save

src/entity.ts

@@ -1000,16 +1000,6 @@ export namespace entity {
  const MAX_DATASTORE_VALUE_LENGTH = 1500;
  if (Array.isArray(entities)) {
  for (const entry of entities) {
- if (entry && entry.name && entry.value) {
- if (
- is.string(entry.value) &&
- Buffer.from(entry.value).length > MAX_DATASTORE_VALUE_LENGTH
- ) {
- entry.excludeFromIndexes = true;
- } else {
- continue;
- }
- }
  findLargeProperties_(entry, path.concat('[]'), properties);
  }
  } else if (is.object(entities)) {

src/index.ts

@@ -39,7 +39,7 @@
 import * as is from 'is';
 import {Transform, pipeline} from 'stream';
 
 import {entity, Entities, Entity, EntityProto, ValueProto} from './entity';
 import {AggregateField} from './aggregate';
 import Key = entity.Key;
 export {Entity, Key, AggregateField};
@@ -68,9 +68,10 @@
 import {google} from '../protos/protos';
 import {AggregateQuery} from './aggregate';
 import {SaveEntity} from './interfaces/save';
+import {extendExcludeFromIndexes} from './utils/entity/extendExcludeFromIndexes';
+import {buildEntityProto} from './utils/entity/buildEntityProto';
 
 const {grpc} = new GrpcClient();
-const addExcludeFromIndexes = entity.addExcludeFromIndexes;
 
 export type PathType = string | number | entity.Int;
 export interface BooleanObject {
@@ -1098,7 +1099,6 @@
  .map(DatastoreRequest.prepareEntityObject_)
  .forEach((entityObject: Entity, index: number) => {
  const mutation: Mutation = {};
- let entityProto: EntityProto = {};
  let method = 'upsert';
 
  if (entityObject.method) {
@@ -1115,70 +1115,8 @@
  insertIndexes[index] = true;
  }
 
- if (Array.isArray(entityObject.data)) {
- // This code populates the excludeFromIndexes list with the right values.
- if (entityObject.excludeLargeProperties) {
- entityObject.data.forEach(
- (data: {
- name: {
- toString(): string;
- };
- value: Entity;
- excludeFromIndexes?: boolean;
- }) => {
- entityObject.excludeFromIndexes = entity.findLargeProperties_(
- data.value,
- data.name.toString(),
- entityObject.excludeFromIndexes
- );
- }
- );
- }
- // This code builds the right entityProto from the entityObject
- entityProto.properties = entityObject.data.reduce(
- (
- acc: EntityProtoReduceAccumulator,
- data: EntityProtoReduceData
- ) => {
- const value = entity.encodeValue(
- data.value,
- data.name.toString()
- );
-
- if (typeof data.excludeFromIndexes === 'boolean') {
- const excluded = data.excludeFromIndexes;
- let values = value.arrayValue && value.arrayValue.values;
-
- if (values) {
- values = values.map((x: ValueProto) => {
- x.excludeFromIndexes = excluded;
- return x;
- });
- } else {
- value.excludeFromIndexes = data.excludeFromIndexes;
- }
- }
-
- acc[data.name] = value;
-
- return acc;
- },
- {}
- );
- // This code adds excludeFromIndexes in the right places
- addExcludeFromIndexes(entityObject.excludeFromIndexes, entityProto);
- } else {
- // This code populates the excludeFromIndexes list with the right values.
- if (entityObject.excludeLargeProperties) {
- entityObject.excludeFromIndexes = entity.findLargeProperties_(
- entityObject.data,
- '',
- entityObject.excludeFromIndexes
- );
- }
- // This code builds the right entityProto from the entityObject
- entityProto = entity.entityToEntityProto(entityObject);
- }
+ extendExcludeFromIndexes(entityObject);
+ const entityProto = buildEntityProto(entityObject);
 
  entityProto.key = entity.keyToKeyProto(entityObject.key);
 

src/utils/entity/buildEntityProto.ts

@@ -0,0 +1,60 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {entity, Entity, EntityProto, ValueProto} from '../../entity';
+import {EntityProtoReduceAccumulator, EntityProtoReduceData} from '../../index';
+import addExcludeFromIndexes = entity.addExcludeFromIndexes;
+
+/**
+ * This function builds the entity proto from the entity object. We cannot
+ * rely on entity.entityToEntityProto for this because this function is only
+ * designed to be used for non-array entities.
+ *
+ */
+export function buildEntityProto(entityObject: Entity) {
+ let entityProto: EntityProto = {};
+ if (Array.isArray(entityObject.data)) {
+ // This code builds the right entityProto from the entityObject
+ entityProto.properties = entityObject.data.reduce(
+ (acc: EntityProtoReduceAccumulator, data: EntityProtoReduceData) => {
+ const value = entity.encodeValue(data.value, data.name.toString());
+
+ if (typeof data.excludeFromIndexes === 'boolean') {
+ const excluded = data.excludeFromIndexes;
+ let values = value.arrayValue && value.arrayValue.values;
+
+ if (values) {
+ values = values.map((x: ValueProto) => {
+ x.excludeFromIndexes = excluded;
+ return x;
+ });
+ } else {
+ value.excludeFromIndexes = data.excludeFromIndexes;
+ }
+ }
+
+ acc[data.name] = value;
+
+ return acc;
+ },
+ {}
+ );
+ // This code adds excludeFromIndexes in the right places
+ addExcludeFromIndexes(entityObject.excludeFromIndexes, entityProto);
+ } else {
+ // This code builds the right entityProto from the entityObject
+ entityProto = entity.entityToEntityProto(entityObject);
+ }
+ return entityProto;
+}

src/utils/entity/extendExcludeFromIndexes.ts

@@ -0,0 +1,52 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {entity, Entity} from '../../entity';
+
+/**
+ * This function extends the excludeFromIndexes list when it finds
+ * large properties in the entity object. The extended excludeFromIndexes
+ * list is then used when building the entity proto.
+ *
+ * @param {Entity} entityObject The entity object to parse for properties to
+ * add to the excludeFromIndexes list.
+ */
+export function extendExcludeFromIndexes(entityObject: Entity) {
+ if (entityObject.excludeLargeProperties) {
+ if (Array.isArray(entityObject.data)) {
+ // This code populates the excludeFromIndexes list with the right values.
+ entityObject.data.forEach(
+ (data: {
+ name: {
+ toString(): string;
+ };
+ value: Entity;
+ excludeFromIndexes?: boolean;
+ }) => {
+ entityObject.excludeFromIndexes = entity.findLargeProperties_(
+ data.value,
+ data.name.toString(),
+ entityObject.excludeFromIndexes
+ );
+ }
+ );
+ } else {
+ entityObject.excludeFromIndexes = entity.findLargeProperties_(
+ entityObject.data,
+ '',
+ entityObject.excludeFromIndexes
+ );
+ }
+ }
+}

system-test/datastore.ts

@@ -348,6 +348,37 @@ async.each(
  await datastore.delete(postKey);
  });
 
+ it('should save a nested name/value pair with name as a long string', async () => {
+ const longString = Buffer.alloc(1501, '.').toString();
+ const postKey = datastore.key(['Post', 'post1']);
+ await datastore.save({
+ key: postKey,
+ data: {
+ metadata: [
+ {
+ name: longString,
+ value: 'some-value',
+ },
+ ],
+ },
+ excludeLargeProperties: true,
+ });
+ });
+ it('should save a nested name/value pair with value as a long string', async () => {
+ const longString = Buffer.alloc(1501, '.').toString();
+ const postKey = datastore.key(['Post', 'post1']);
+ await datastore.save({
+ key: postKey,
+ data: [
+ {
+ name: 'some-name',
+ value: longString,
+ },
+ ],
+ excludeLargeProperties: true,
+ });
+ });
+
  describe('multi-db support for read and write operations', () => {
  const namespace = `${Date.now()}`;
  const keyHierarchy = ['Post', 'post1'];