Add support for remembering a position within the input stream (879255e2) · Commits · Legacy / gitolite / user / sebastian / Trunnel

doc/trunnel.md

+22 −0

Original line number	Diff line number	Diff line
		@@ -281,6 +281,28 @@ In newly constructed structures, all variable-length arrays are empty.
		It's an error to try to encode a variable-length array with a length field if
		that array's length field doesn't match its actual length.

		### Structure members: zero-length indices into the input

		Sometimes you need to record the position in the input the corresponds to
		a position in the structure. You can use an `@ptr` field to record
		a position within a structure when parsing it:

		struct s {
		nulterm unsigned_header;
		@ptr start_of_signed_material;
		u32 bodylen;
		u8 body[bodylen];
		u64 flags;
		@ptr end_of_signed_material;
		u16 signature_len;
		u8 signature[signature_len];
		}

		When an object of this type is parsed, then `start_of_signed_material`
		and `end_of_signed_material` will get set to pointers into the input.
		These pointers are only set when the input is parsed; you don't need
		to set them to encode the object.

		### Structure members: unions

		You can specify that different elements should be parsed based on some

lib/trunnel/CodeGen.py

+37 −0

Original line number	Diff line number	Diff line
		@@ -338,6 +338,9 @@ class Checker(ASTVisitor):
		def visitSMString(self, sms):
		self.addMemberName(sms.name)

		def visitSMPosition(self, smp):
		self.addMemberName(smp.name)

		def visitSMLenConstrained(self, sml):
		if sml.lengthfield != None:
		self.checkIntField(
		@@ -544,6 +547,9 @@ class Annotator(ASTVisitor):
		def visitSMString(self, ss):
		self.annotateMember(ss)

		def visitSMPosition(self, smp):
		self.annotateMember(smp)

		def visitSMLenConstrained(self, sml):
		sml.lengthfieldmember = None
		if sml.lengthfield is not None:
		@@ -781,6 +787,11 @@ class DeclarationGenerationVisitor(CodeGenerator):

		self.w("char *%s;\n" % (ss.c_name))

		def visitSMPosition(self, smp):
		if smp.annotation != None:
		self.w(smp.annotation)
		self.w("const uint8_t *%s;\n" % smp.c_name)

		def visitSMLenConstrained(self, sml):
		sml.visitChildren(self)

		@@ -1072,6 +1083,9 @@ class FreeFnGenerator(CodeGenerator):
		self.w("trunnel_wipestr(obj->%s);\n" % (ss.c_name))
		self.w("trunnel_free(obj->%s);\n" % (ss.c_name))

		def visitSMPosition(self, smp):
		pass

		def visitSMLenConstrained(self, sml):
		sml.visitChildren(self)

		@@ -1579,6 +1593,17 @@ class AccessorFnGenerator(CodeGenerator):
		return 0;
		}}""", c_name=sms.c_name)

		def visitSMPosition(self, smp):
		st = self.structName
		nm = smp.c_fn_name
		self.docstring("Return the position for %s when we parsed "
		"this object"%nm)
		self.declaration("const uint8_t *",
		"%s_get_%s(const %s_t *inp)" % (st,nm,st))
		self.format("""
		{{
		return inp->{nm};
		}}""", nm = smp.c_name)

		def iterateOverFixedArray(generator, sfa, body, extraDecl=""):
		"""Helper: write the code needed to iterate over every element of a
		@@ -1739,6 +1764,9 @@ class CheckFnGenerator(CodeGenerator):
		self.w('if (NULL == obj->%s)\n return "Missing %s";\n' %
		(ss.c_name, ss.c_name))

		def visitSMPosition(self, smp):
		pass

		def visitSMLenConstrained(self, sml):
		# To check a SMlenConstrained, check its children.
		sml.visitChildren(self)
		@@ -1872,6 +1900,9 @@ class EncodedLenFnGenerator(CodeGenerator):
		self.eltHeader(ss)
		self.w("result += strlen(obj->%s) + 1;\n" % ss.c_name)

		def visitSMPosition(self, smp):
		pass

		def visitSMLenConstrained(self, sml):
		sml.visitChildren(self)

		@@ -2191,6 +2222,9 @@ class EncodeFnGenerator(CodeGenerator):
		ptr += len + 1; written += len + 1;
		}}""", c_name=ss.c_name)

		def visitSMPosition(self, smp):
		pass

		def visitSMLenConstrained(self, sml):
		# To encode a length-constained field of a structure,
		# remember the position at which we began writing to the union.
		@@ -2643,6 +2677,9 @@ class ParseFnGenerator(CodeGenerator):
		remaining -= memlen; ptr += memlen;
		}}""", c_name=ss.c_name, truncated=self.truncatedLabel)

		def visitSMPosition(self, smp):
		self.format("obj->{c_name} = ptr;", c_name=smp.c_name);

		def visitSMLenConstrained(self, sml):
		# To parse a length-constrained region, make sure that at
		# least that many bytes remain in the structure. Then,

lib/trunnel/Grammar.py

+28 −1

Original line number	Diff line number	Diff line
		@@ -113,7 +113,7 @@ class Lexer(trunnel.spark.GenericScanner, object):
		trunnel.spark.GenericScanner.tokenize(self, input)
		return self.rv

		@pattern(r"(?:[;{}\[\]\-=,:]\|\.\.\.\|\.\.\|\.)")
		@pattern(r"(?:[;{}@\[\]\-=,:]\|\.\.\.\|\.\.\|\.)")
		def t_punctuation(self, s):
		self.rv.append(Token(s, self.lineno))

		@@ -570,6 +570,15 @@ class SMIgnore(StructMember):
		ignored."""
		pass

		class SMPosition(StructMember):
		""" A struct member: notes that we should store a pointer to this point
		in the input when we """
		def __init__(self, name):
		StructMember.__init__(self, name)

		def __str__(self):
		return "@" + self.name


		class IDReference(AST):

		@@ -778,6 +787,10 @@ class Parser(trunnel.spark.GenericParser, object):
		def p_StructMember_4(self, info):
		return info[0]

		@rule(" StructMember ::= SMPosition ")
		def p_StructMember_5(self, info):
		return info[0]

		@rule(" SMInteger ::= IntType ID OptIntConstraint ")
		def p_SMInteger(self, info):
		return SMInteger(info[0], str(info[1]), info[2])
		@@ -997,6 +1010,10 @@ class Parser(trunnel.spark.GenericParser, object):
		def p_UnionField_5(self, info):
		return info[0]

		@rule(" UnionField ::= SMSPosition ")
		def p_UnionField_6(self, info):
		return info[0]

		@rule(" ContextDecl ::= context ID { ContextMembers } ")
		def p_ContextDecl(self, info):
		return StructDecl(str(info[1]), info[3], isContext=True)
		@@ -1017,6 +1034,16 @@ class Parser(trunnel.spark.GenericParser, object):
		def p_ContextMember(self, info):
		return SMInteger(info[0], str(info[1]), None)

		@rule(" SMPosition ::= @ PtrKW ID ")
		def p_SMPosition(self, info):
		return SMPosition(str(info[2]))

		@rule(" PtrKW ::= ID ")
		def p_PtrKW(self, info):
		if str(info[0]) != 'ptr':
		raise SyntaxError("Expected 'ptr' at %s" % info[0].lineno)
		return None

		if __name__ == '__main__':
		print ("===== Here is our actual grammar, extracted from Grammar.py\n")

test/Makefile

+6 −0

Original line number	Diff line number	Diff line
		@@ -31,6 +31,7 @@ TEST_OBJS = \
		c/test_contexts_varsize2.o \
		c/test_contexts_complex.o \
		c/test_remainder_repeats.o \
		c/test_positions.o \
		c/test_util.o

		BOILERPLATE_FILES=\
		@@ -44,6 +45,7 @@ OBJS=tinytest/tinytest.o \
		valid/opaque.o \
		valid/leftover.o \
		valid/contexts.o \
		valid/positions.o \
		./include/trunnel.o \
		$(TEST_OBJS)

		@@ -69,6 +71,7 @@ valid/derived.o: valid/derived.h valid/derived.c
		valid/opaque.o: valid/opaque.h valid/opaque.h
		valid/leftover.o: valid/leftover.h valid/leftover.c
		valid/contexts.o: valid/contexts.h
		valid/positions.o: valid/positions.h
		$(TEST_OBJS) : tinytest/tinytest.h tinytest/tinytest_macros.h valid/simple.h valid/derived.h
		$(OBJS) : include/trunnel.h include/trunnel-impl.h
		tinytest/tinytest.o: tinytest/tinytest.h tinytest/tinytest_macros.h
		@@ -88,5 +91,8 @@ valid/leftover.c valid/leftover.h: valid/leftover.trunnel ../lib/trunnel/*py
		valid/contexts.c valid/contexts.h: valid/contexts.trunnel ../lib/trunnel/*py
		PYTHONPATH=../lib:${PYTHONPATH} python -m trunnel valid/contexts.trunnel

		valid/positions.c valid/positions.h: valid/positions.trunnel ../lib/trunnel/*py
		PYTHONPATH=../lib:${PYTHONPATH} python -m trunnel valid/positions.trunnel

		$(BOILERPLATE_FILES): ../lib/trunnel/py ../lib/trunnel/data/.[ch]
		PYTHONPATH=../lib:${PYTHONPATH} python -m trunnel --target-dir=./include --write-c-files

test/c/test.c

+1 −0

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@ struct testgroup_t test_groups[] = {
		{ "contexts/varsize/", contexts_varsize_tests },
		{ "contexts/varsize2/", contexts_varsize2_tests },
		{ "contexts/complex/", contexts_complex_tests },
		{ "positions/", positions_tests },
		END_OF_GROUPS,
		};