Expose flags on every node type

Author: kddnewton

config.yml

@@ -117,6 +117,7 @@ Each node is structured like the following table:
 | --- | --- |
 | `1` | node type |
 | location | node location |
+| varuint | node flags |
 
 Every field on the node is then appended to the serialized string. The fields can be determined by referencing `config.yml`. Depending on the type of field, it could take a couple of different forms, described below:
 

docs/serialization.md

@@ -21,7 +21,10 @@ module RegularExpressionOptions # :nodoc:
  # Returns a numeric value that represents the flags that were used to create
  # the regular expression.
  def options
- o = flags & (RegularExpressionFlags::IGNORE_CASE | RegularExpressionFlags::EXTENDED | RegularExpressionFlags::MULTI_LINE)
+ o = 0
+ o |= Regexp::IGNORECASE if flags.anybits?(RegularExpressionFlags::IGNORE_CASE)
+ o |= Regexp::EXTENDED if flags.anybits?(RegularExpressionFlags::EXTENDED)
+ o |= Regexp::MULTILINE if flags.anybits?(RegularExpressionFlags::MULTI_LINE)
  o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8)
  o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT)
  o
@@ -87,6 +90,7 @@ def to_interpolated
  InterpolatedXStringNode.new(
  source,
  location,
+ flags,
  opening_loc,
  [StringNode.new(source, content_loc, 0, nil, content_loc, nil, unescaped)],
  closing_loc
@@ -117,7 +121,7 @@ def numeric
  if denominator == 1
  IntegerNode.new(source, location.chop, flags, numerator)
  else
- FloatNode.new(source, location.chop, numerator.to_f / denominator)
+ FloatNode.new(source, location.chop, 0, numerator.to_f / denominator)
  end
  end
  end
@@ -195,7 +199,7 @@ def full_name
  # continue to supply that API.
  def child
  deprecated("name", "name_loc")
- name ? ConstantReadNode.new(source, name_loc, name) : MissingNode.new(source, location)
+ name ? ConstantReadNode.new(source, name_loc, 0, name) : MissingNode.new(source, location, 0)
  end
  end
 
@@ -231,7 +235,7 @@ def full_name
  # continue to supply that API.
  def child
  deprecated("name", "name_loc")
- name ? ConstantReadNode.new(source, name_loc, name) : MissingNode.new(source, location)
+ name ? ConstantReadNode.new(source, name_loc, 0, name) : MissingNode.new(source, location, 0)
  end
  end
 

lib/prism/node_ext.rb

@@ -45,7 +45,7 @@ def visit_block_node(node)
 
  # Mark if/unless nodes as newlines.
  def visit_if_node(node)
- node.newline!(@lines)
+ node.newline_flag!(@lines)
  super(node)
  end
 
@@ -54,101 +54,101 @@ def visit_if_node(node)
  # Permit statements lists to mark newlines within themselves.
  def visit_statements_node(node)
  node.body.each do |child|
- child.newline!(@lines)
+ child.newline_flag!(@lines)
  end
  super(node)
  end
  end
  end
 
  class Node
- def newline? # :nodoc:
- @newline ? true : false
+ def newline_flag? # :nodoc:
+ @newline_flag ? true : false
  end
 
- def newline!(lines) # :nodoc:
+ def newline_flag!(lines) # :nodoc:
  line = location.start_line
  unless lines[line]
  lines[line] = true
- @newline = true
+ @newline_flag = true
  end
  end
  end
 
  class BeginNode < Node
- def newline!(lines) # :nodoc:
+ def newline_flag!(lines) # :nodoc:
  # Never mark BeginNode with a newline flag, mark children instead.
  end
  end
 
  class ParenthesesNode < Node
- def newline!(lines) # :nodoc:
+ def newline_flag!(lines) # :nodoc:
  # Never mark ParenthesesNode with a newline flag, mark children instead.
  end
  end
 
  class IfNode < Node
- def newline!(lines) # :nodoc:
- predicate.newline!(lines)
+ def newline_flag!(lines) # :nodoc:
+ predicate.newline_flag!(lines)
  end
  end
 
  class UnlessNode < Node
- def newline!(lines) # :nodoc:
- predicate.newline!(lines)
+ def newline_flag!(lines) # :nodoc:
+ predicate.newline_flag!(lines)
  end
  end
 
  class UntilNode < Node
- def newline!(lines) # :nodoc:
- predicate.newline!(lines)
+ def newline_flag!(lines) # :nodoc:
+ predicate.newline_flag!(lines)
  end
  end
 
  class WhileNode < Node
- def newline!(lines) # :nodoc:
- predicate.newline!(lines)
+ def newline_flag!(lines) # :nodoc:
+ predicate.newline_flag!(lines)
  end
  end
 
  class RescueModifierNode < Node
- def newline!(lines) # :nodoc:
- expression.newline!(lines)
+ def newline_flag!(lines) # :nodoc:
+ expression.newline_flag!(lines)
  end
  end
 
  class InterpolatedMatchLastLineNode < Node
- def newline!(lines) # :nodoc:
+ def newline_flag!(lines) # :nodoc:
  first = parts.first
- first.newline!(lines) if first
+ first.newline_flag!(lines) if first
  end
  end
 
  class InterpolatedRegularExpressionNode < Node
- def newline!(lines) # :nodoc:
+ def newline_flag!(lines) # :nodoc:
  first = parts.first
- first.newline!(lines) if first
+ first.newline_flag!(lines) if first
  end
  end
 
  class InterpolatedStringNode < Node
- def newline!(lines) # :nodoc:
+ def newline_flag!(lines) # :nodoc:
  first = parts.first
- first.newline!(lines) if first
+ first.newline_flag!(lines) if first
  end
  end
 
  class InterpolatedSymbolNode < Node
- def newline!(lines) # :nodoc:
+ def newline_flag!(lines) # :nodoc:
  first = parts.first
- first.newline!(lines) if first
+ first.newline_flag!(lines) if first
  end
  end
 
  class InterpolatedXStringNode < Node
- def newline!(lines) # :nodoc:
+ def newline_flag!(lines) # :nodoc:
  first = parts.first
- first.newline!(lines) if first
+ first.newline_flag!(lines) if first
  end
  end
 end

lib/prism/parse_result/newlines.rb

@@ -1,8 +1,8 @@
 module Prism
  class Node
- @newline: bool
+ @newline_flag: bool
 
- def newline?: () -> bool
- def set_newline_flag: (Array[bool]) -> void
+ def newline_flag?: () -> bool
+ def newline_flag!: (Array[bool] lines) -> void
  end
 end

sig/prism/_private/node.rbs

@@ -170,17 +170,20 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
  <%- nodes.each do |node| -%>
 #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
  case <%= node.type %>: {
- <%- if node.fields.any? { |field| ![Prism::Template::NodeField, Prism::Template::OptionalNodeField, Prism::Template::FlagsField].include?(field.class) } -%>
+ <%- if node.fields.any? { |field| ![Prism::Template::NodeField, Prism::Template::OptionalNodeField].include?(field.class) } -%>
  pm_<%= node.human %>_t *cast = (pm_<%= node.human %>_t *) node;
  <%- end -%>
- VALUE argv[<%= node.fields.length + 2 %>];
+ VALUE argv[<%= node.fields.length + 3 %>];
 
  // source
  argv[0] = source;
 
  // location
  argv[1] = pm_location_new(parser, node->location.start, node->location.end);
- <%- node.fields.each.with_index(2) do |field, index| -%>
+
+ // flags
+ argv[2] = ULONG2NUM(node->flags);
+ <%- node.fields.each.with_index(3) do |field, index| -%>
 
  // <%= field.name %>
  <%- case field -%>
@@ -221,9 +224,6 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
  <%- when Prism::Template::UInt32Field -%>
 #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
  argv[<%= index %>] = ULONG2NUM(cast-><%= field.name %>);
- <%- when Prism::Template::FlagsField -%>
-#line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
- argv[<%= index %>] = ULONG2NUM(node->flags & ~PM_NODE_FLAG_COMMON_MASK);
  <%- when Prism::Template::IntegerField -%>
 #line <%= __LINE__ + 1 %> "prism/templates/ext/prism/<%= File.basename(__FILE__) %>"
  argv[<%= index %>] = pm_integer_new(&cast-><%= field.name %>);
@@ -235,7 +235,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
  <%- end -%>
  <%- end -%>
 
- rb_ary_push(value_stack, rb_class_new_instance(<%= node.fields.length + 2 %>, argv, rb_cPrism<%= node.name %>));
+ rb_ary_push(value_stack, rb_class_new_instance(<%= node.fields.length + 3 %>, argv, rb_cPrism<%= node.name %>));
  break;
  }
  <%- end -%>

templates/ext/prism/api_node.c.erb

@@ -100,11 +100,8 @@ typedef uint16_t pm_node_flags_t;
  * We store the flags enum in every node in the tree. Some flags are common to
  * all nodes (the ones listed below). Others are specific to certain node types.
  */
-#define PM_NODE_FLAG_BITS (sizeof(pm_node_flags_t) * 8)
-
-static const pm_node_flags_t PM_NODE_FLAG_NEWLINE = (1 << (PM_NODE_FLAG_BITS - 1));
-static const pm_node_flags_t PM_NODE_FLAG_STATIC_LITERAL = (1 << (PM_NODE_FLAG_BITS - 2));
-static const pm_node_flags_t PM_NODE_FLAG_COMMON_MASK = (1 << (PM_NODE_FLAG_BITS - 1)) | (1 << (PM_NODE_FLAG_BITS - 2));
+static const pm_node_flags_t PM_NODE_FLAG_NEWLINE = 0x1;
+static const pm_node_flags_t PM_NODE_FLAG_STATIC_LITERAL = 0x2;
 
 /**
  * Cast the type to an enum to allow the compiler to provide exhaustiveness
@@ -151,11 +148,10 @@ typedef struct pm_node {
  * <%= node.name %>
  *
  * Type: <%= node.type %>
-<%- if (node_flags = node.fields.find { |field| field.is_a? Prism::Template::FlagsField }) -%>
+<%- if (node_flags = node.flags) -%>
  * Flags:
-<%- found = flags.find { |flag| flag.name == node_flags.kind }.tap { |found| raise "Expected to find #{field.kind}" unless found } -%>
-<%- found.values.each do |value| -%>
- * PM_<%= found.human.upcase %>_<%= value.name %>
+<%- node_flags.values.each do |value| -%>
+ * PM_<%= node_flags.human.upcase %>_<%= value.name %>
 <%- end -%>
 <%- end -%>
  *
@@ -164,7 +160,7 @@ typedef struct pm_node {
 typedef struct pm_<%= node.human %> {
  /** The embedded base node. */
  pm_node_t base;
-<%- node.fields.grep_v(Prism::Template::FlagsField).each do |field| -%>
+<%- node.fields.each do |field| -%>
 
  /**
  * <%= node.name %>#<%= field.name %>
@@ -201,7 +197,7 @@ typedef enum pm_<%= flag.human %> {
  <%- flag.values.each_with_index do |value, index| -%>
 <%= "\n" if index > 0 -%>
  /** <%= value.comment %> */
- PM_<%= flag.human.upcase %>_<%= value.name %> = <%= 1 << index %>,
+ PM_<%= flag.human.upcase %>_<%= value.name %> = <%= 1 << (index + 2) %>,
  <%- end -%>
 } pm_<%= flag.human %>_t;
 <%- end -%>

templates/include/prism/ast.h.erb

@@ -362,6 +362,7 @@ public class Loader {
  case <%= index + 1 %>:
  <%-
  params = node.needs_serialized_length? ? ["buffer.getInt()"] : []
+ params << "loadFlags()"
  params.concat node.semantic_fields.map { |field|
  case field
  when Prism::Template::NodeField then "#{field.java_cast}loadNode()"
@@ -378,7 +379,6 @@ public class Loader {
  when Prism::Template::OptionalLocationField then "loadOptionalLocation()"
  when Prism::Template::UInt8Field then "buffer.get()"
  when Prism::Template::UInt32Field then "loadVarUInt()"
- when Prism::Template::FlagsField then "loadFlags()"
  when Prism::Template::IntegerField then "loadInteger()"
  when Prism::Template::DoubleField then "buffer.getDouble()"
  else raise

templates/java/org/prism/Loader.java.erb

@@ -220,25 +220,28 @@ public abstract class Nodes {
  <%- end -%>
 
  <%-
- params = node.needs_serialized_length? ? ["int serializedLength"] : []
+ params = []
+ params << "int serializedLength" if node.needs_serialized_length?
+ params << "short flags"
  params.concat node.semantic_fields.map { |field| "#{field.java_type} #{field.name}" }
  params.concat ["int startOffset", "int length"]
  -%>
- public <%=node.name -%>(<%= params.join(", ") %>) {
+ public <%= node.name -%>(<%= params.join(", ") %>) {
  super(startOffset, length);
  <%- if node.needs_serialized_length? -%>
  this.serializedLength = serializedLength;
  <%- end -%>
+ this.flags = flags;
  <%- node.semantic_fields.each do |field| -%>
  this.<%= field.name %> = <%= field.name %>;
  <%- end -%>
  }
  <%# methods for flags -%>
- <%- node.semantic_fields.grep(Prism::Template::FlagsField).each do |field| -%>
- <%- flags.find { |flag| flag.name == field.kind }.tap { |flag| raise "Expected to find #{field.kind}" unless flag }.values.each do |value| -%>
+ <%- if (node_flags = node.flags) -%>
+ <%- node_flags.values.each do |value| -%>
 
  public boolean is<%= value.camelcase %>() {
- return <%= field.kind %>.is<%= value.camelcase %>(this.<%= field.name %>);
+ return <%= node_flags.name %>.is<%= value.camelcase %>(this.<%= node_flags.name %>);
  }
  <%- end -%>
  <%- end -%>
@@ -321,7 +324,7 @@ public abstract class Nodes {
  <%- if node.fields.any?(Prism::Template::NodeListField) or node.fields.any?(Prism::Template::ConstantListField) -%>
  String nextNextIndent = nextIndent + " ";
  <%- end -%>
- <%- node.fields.grep_v(Prism::Template::LocationField).grep_v(Prism::Template::OptionalLocationField).each do |field| -%>
+ <%- [*node.flags, *node.fields.grep_v(Prism::Template::LocationField).grep_v(Prism::Template::OptionalLocationField)].each do |field| -%>
  builder.append(nextIndent);
  builder.append("<%= field.name %>: ");
  <%- case field -%>
@@ -348,7 +351,10 @@ public abstract class Nodes {
  for (<%= string_type %> constant : this.<%= field.name %>) {
  builder.append(nextNextIndent).append('"').append(constant).append('"').append('\n');
  }
- <%- when Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::FlagsField, Prism::Template::IntegerField, Prism::Template::DoubleField -%>
+ <%- when Prism::Template::Flags -%>
+ builder.append(this.flags);
+ builder.append('\n');
+ <%- when Prism::Template::UInt8Field, Prism::Template::UInt32Field, Prism::Template::IntegerField, Prism::Template::DoubleField -%>
  builder.append(this.<%= field.name %>);
  builder.append('\n');
  <%- else -%>

templates/java/org/prism/Nodes.java.erb

@@ -303,7 +303,7 @@ export function deserialize(source, array) {
  <%- if node.needs_serialized_length? -%>
  buffer.readUint32();
  <%- end -%>
- return new nodes.<%= node.name %>(<%= (node.fields.map { |field|
+ return new nodes.<%= node.name %>(<%= ["location", "buffer.readVarInt()", *node.fields.map { |field|
  case field
  when Prism::Template::NodeField then "readRequiredNode()"
  when Prism::Template::OptionalNodeField then "readOptionalNode()"
@@ -315,12 +315,11 @@ export function deserialize(source, array) {
  when Prism::Template::LocationField then "buffer.readLocation()"
  when Prism::Template::OptionalLocationField then "buffer.readOptionalLocation()"
  when Prism::Template::UInt8Field then "buffer.readByte()"
- when Prism::Template::UInt32Field, Prism::Template::FlagsField then "buffer.readVarInt()"
+ when Prism::Template::UInt32Field then "buffer.readVarInt()"
  when Prism::Template::IntegerField then "readInteger()"
  when Prism::Template::DoubleField then "buffer.readDouble()"
- else raise
  end
- } + ["location"]).join(", ") -%>);
+ }].join(", ") -%>);
  <%- end -%>
  default:
  throw new Error(`Unknown node type: ${type}`);