Increasing num shards allowed (#388)

* Removed test on too many shards

Author: govindr-cm

ciphercore-base/src/graphs.rs

@@ -2450,20 +2450,7 @@ impl Graph {
  /// Adds a node that computes sharding of a given table according to a given sharding config.
  /// Sharding config contains names of the columns whose hashed values are used for sharding.
  /// The size of each shard (i.e., the number of rows) and the number of shards is given in the sharding config.
- /// The number of shards should be smaller than 700.
- ///
- ///
- /// If some resulting shards don't have `shard_size` elements, they're padded with zeros to reach this size.
- /// If the size of some shards exceeds `shard_size`, sharding fails.
- ///
- /// To choose these parameters, consult [the following paper](http://wwwmayr.informatik.tu-muenchen.de/personen/raab/publ/balls.pdf).
- /// Note that for large shard sizes and small number of shards, it holds that
- ///
- /// `shard_size = num_input_rows / num_shards + alpha * sqrt(2 * num_input_rows / num_shards * log(num_shards))`.
- ///
- /// With `alpha = 2`, it is possible to achieve failure probability 2^(-40) if `num_shards < 700` and `shard_size > 2^17`.
- ///
- ///
+
  /// Each shard is accompanied by a Boolean mask indicating whether a corresponding row stems from the input table or padded (1 if a row comes from input).
  /// The output is given in the form of a tuple of `(mask, shard)`, where `mask` is a binary array and `shard` is a table, i.e., named tuple.
  ///

ciphercore-base/src/type_inference.rs

@@ -1574,13 +1574,6 @@ impl TypeInferenceWorker {
  check_table_and_extract_column_types(input_t.clone(), false, false)?;
  let headers: Vec<String> = headers_types.keys().cloned().collect();
 
- if shard_config.num_shards > 700 {
- return Err(runtime_error!(
- "No more than 700 shards can be handled, {} provided",
- shard_config.num_shards
- ));
- }
-
  let num_elements_in_all_shards = shard_config.shard_size * shard_config.num_shards;
  if num_entries > num_elements_in_all_shards {
  return Err(runtime_error!("Input elements can't fit given shards. Shards can contain {} elements, while input has {}", num_elements_in_all_shards, num_entries));
@@ -4771,17 +4764,6 @@ mod tests {
  shard_headers: vec!["ID".to_owned(), "ID".to_owned()],
  },
  )?;
- test_shard_fail(
- named_tuple_type(vec![
- ("Income".to_owned(), array_type(vec![20], UINT64)),
- ("ID".to_owned(), array_type(vec![20], UINT64)),
- ]),
- ShardConfig {
- num_shards: 800,
- shard_size: 10,
- shard_headers: vec!["ID".to_owned()],
- },
- )?;
 
  Ok(())
  }