Sign in to contribute to source. how it works
Component parsing.TypeParser by barry
//parse a source file into the below using string syntax rules (isInterface() etc.); then scan for block comments just before interface or function header definitions and extract JSON from them (maybe consult the JSON parser as a really good example of strict parsing)
const int TOKEN_UNKNOWN = 0
const int TOKEN_LINE_COMMENT = 1
const int TOKEN_BLOCK_COMMENT = 2
const int TOKEN_INTERFACE = 3
const int TOKEN_FUNCTION_HEADER = 4
data Token {
int type
char content[]
int sourceLength
}
data IncludeList{
String list[]
}
component provides TypeParser requires io.Output out, io.FileSystem fs, io.File, data.json.JSONParser parser, data.StringUtil stringUtil, parsing.Tokeniser, data.IntUtil iu, System sysInfo, data.query.Search search {
SourceFile primaryFiles[]
SourceFile supportFiles[]
String builtinTypes[]
String parseTokens[]
String parseKeywords[]
Tokeniser tokeniser
void initialiseParseTokens()
{
parseTokens = new String[](new String(":"),
new String(";"),
new String(","),
new String("."),
new String("!"),
new String(">"),
new String("<"),
new String("++"),
new String("--"),
new String("+="),
new String("-="),
new String("+"),
new String("-"),
new String("/"),
new String("*"),
new String("&&"),
new String("||"),
new String("&"),
new String("|"),
new String("==="),
new String("!=="),
new String("=="),
new String("!="),
new String("="),
new String("{"),
new String("}"),
new String("["),
new String("]"),
new String("("),
new String(")"))
}
bool isToken(char str[])
{
for (int i = 0; i < parseTokens.arrayLength; i++)
{
if (str == parseTokens[i].string)
return true
}
return false
}
void initialiseParseKeywords()
{
parseKeywords = new String[](new String("component"),
new String("interface"),
new String("data"),
new String("extends"),
new String("provides"),
new String("requires"),
new String("const"),
new String("transfer"),
new String("static"),
new String("event"),
new String("nocycle"),
new String("new"),
new String("true"),
new String("false"),
new String("null"),
new String("uses"),
new String("implementation"),
new String("if"),
new String("while"),
new String("for"),
new String("else"),
new String("break"),
new String("throw"),
new String("return"))
}
bool isKeyword(char str[])
{
for (int i = 0; i < parseKeywords.arrayLength; i++)
{
if (str == parseKeywords[i].string)
return true
}
return false
}
void initialiseBuiltinTypes()
{
builtinTypes = new String[](new String("void"),
new String("char"),
new String("byte"),
new String("bool"),
new String("TypeField"),
new String("int"),
new String("int1"),
new String("int2"),
new String("int4"),
new String("int8"),
new String("int16"),
new String("int32"),
new String("int64"),
new String("int128"),
new String("int256"),
new String("int512"),
new String("dec"),
new String("dec1"),
new String("dec2"),
new String("dec4"),
new String("dec8"),
new String("dec16"),
new String("dec32"),
new String("dec64"),
new String("dec128"),
new String("dec256"),
new String("dec512"),
new String("Data")
)
}
bool isBuiltinType(char str[])
{
for (int i = 0; i < builtinTypes.arrayLength; i++)
{
if (str == builtinTypes[i].string)
return true
}
return false
}
bool isUses(ParseToken tokens[], int start)
{
if (tokens[start].type == ParseToken.TYPE_PARTICLE && tokens[start].content == "uses")
{
if (start + 1 < tokens.arrayLength && tokens[start+1].type == ParseToken.TYPE_PARTICLE)
{
return true
}
}
return false
}
int consumeUses(ParseToken tokens[], int start, IncludeList incList)
{
if (tokens[start].type == ParseToken.TYPE_PARTICLE && tokens[start].content == "uses")
{
int count = 1
start ++
while (start < tokens.arrayLength)
{
if (count > 1 && tokens[start].content != ",")
{
break
}
else if (count > 1)
{
start ++
count ++
}
//the uses path, maybe separated by dots
char pkg[]
pkg = new char[](pkg, tokens[start].content)
start ++
count ++
while (tokens[start].content == ".")
{
start ++
count ++
pkg = new char[](pkg, ".", tokens[start].content)
start ++
count ++
}
incList.list = new String[](incList.list, new String(pkg))
}
return count
}
return 0
}
bool isTypeDefinition(ParseToken tokens[], int start)
{
if (tokens[start].type == ParseToken.TYPE_PARTICLE &&
(tokens[start].content == "data" || tokens[start].content == "interface"))
{
start ++
if (start >= tokens.arrayLength)
return false
if (isToken(tokens[start].content))
return false
start ++
if (start >= tokens.arrayLength)
return false
return true
}
return false
}
bool isComponentDefinition(ParseToken tokens[], int start)
{
if (tokens[start].type == ParseToken.TYPE_PARTICLE &&
(tokens[start].content == "component"))
{
start ++
if (start >= tokens.arrayLength)
return false
if (isToken(tokens[start].content))
return false
start ++
if (start >= tokens.arrayLength)
return false
return true
}
return false
}
int skipScope(ParseToken tokens[], int start)
{
int count = 0
if (tokens[start].content == "{")
{
start ++
count ++
int brDepth = 1
while (start < tokens.arrayLength)
{
if (tokens[start].content == "}")
{
brDepth --
if (brDepth == 0)
{
count ++
start ++
break
}
}
else if (tokens[start].content == "{")
{
brDepth ++
}
count ++
start ++
}
}
return count
}
int consumeTypeDefinition(SourceFile sf, char docComment[], ParseToken tokens[], int start, IncludeList incList, String searchPaths[])
{
//collect the type name and also extract any "extends" clause
int count = 0
addType(sf, tokens[start].content, tokens[start+1].content, docComment, searchPaths)
bool isDataDef = tokens[start].content == "data"
bool isIntfDef = tokens[start].content == "interface"
//type keyword
start ++
count ++
//type name
start ++
count ++
//nocycle?
if (tokens[start].content == "nocycle")
{
start ++
count ++
}
//extends?
if (tokens[start].content == "extends")
{
start ++
count ++
//(package) and type name of the thing we're extending
char pkg[]
pkg = new char[](pkg, tokens[start].content)
start ++
count ++
while (tokens[start].content == ".")
{
start ++
count ++
pkg = new char[](pkg, ".", tokens[start].content)
start ++
count ++
}
//check if this type was defined in the current source file; otherwise put it on the include list for search
if ((isDataDef && getDataDef(sf, pkg) == null) || (isIntfDef && getInterfaceDef(sf, null, pkg) == null))
{
incList.list = new String[](incList.list, new String(pkg))
}
}
//nocycle?
if (tokens[start].content == "nocycle")
{
start ++
count ++
}
return count + skipScope(tokens, start)
}
int consumeComponentDefinition(SourceFile sf, char docComment[], ParseToken tokens[], int start, IncludeList incList)
{
int count = 0
//get the provides and requires lists, including semantic variants
ComponentDef cdef = new ComponentDef(class = TypeDef.COMPONENT)
sf.types = new TypeDef[](sf.types, cdef)
//the word "component"
count ++
start ++
//"requires" or "provides"
if (tokens[start].content == "provides")
{
count ++
start ++
//a comma-separated list of types, potentially with package identifiers and semantic variants, until we hit either "requires" or "{"
while (true)
{
char intfType[] = null
char intfVar[] = null
//part of a type name
intfType = new char[](intfType, tokens[start].content)
count ++
start ++
while (tokens[start].content == ".")
{
intfType = new char[](intfType, ".", tokens[start+1].content)
count += 2
start += 2
}
if (tokens[start].content == ":")
{
intfVar = tokens[start+1].content
start += 2
count += 2
}
ProIntfDef newPro = new ProIntfDef(intfType, intfVar)
cdef.provIntfs = new ProIntfDef[](cdef.provIntfs, newPro)
incList.list = new String[](incList.list, new String(intfType))
//brackets? (secondary object-level interfaces)
if (tokens[start].content == "(")
{
start ++
intfType = null
while (tokens[start].content != ")")
{
if (tokens[start].content == ",")
{
if (getInterfaceDef(sf, null, intfType) == null)
incList.list = new String[](incList.list, new String(intfType))
newPro.subInterfaces = new String[](newPro.subInterfaces, new String(intfType))
intfType = null
}
else
{
intfType = new char[](intfType, tokens[start].content)
}
start ++
}
if (intfType != null)
{
if (getInterfaceDef(sf, null, intfType) == null)
incList.list = new String[](incList.list, new String(intfType))
newPro.subInterfaces = new String[](newPro.subInterfaces, new String(intfType))
}
start ++
}
if (tokens[start].content == "requires" || tokens[start].content == "{")
{
break
}
//comma
start ++
count ++
}
}
if (tokens[start].content == "requires")
{
count ++
start ++
//a comma-separated list of types, potentially with package identifiers and semantic variants, until we hit "{"
//a comma-separated list of types, potentially with package identifiers and semantic variants, until we hit either "requires" or "{"
while (true)
{
char intfType[] = null
char intfVar[] = null
ReqIntfDef newReq = new ReqIntfDef()
//native?
if (tokens[start].content == "native")
{
newReq.isNative = true
count ++
start ++
}
//part of a type name
intfType = new char[](intfType, tokens[start].content)
count ++
start ++
while (tokens[start].content == ".")
{
intfType = new char[](intfType, ".", tokens[start+1].content)
count += 2
start += 2
}
if (tokens[start].content == ":")
{
intfVar = tokens[start+1].content
start += 2
count += 2
}
newReq.typeName = intfType
newReq.variantName = intfVar
cdef.reqIntfs = new ReqIntfDef[](cdef.reqIntfs, newReq)
//check if this interface was declared in this file (usually the case for "native" interfaces); if not, search for it
if (getInterfaceDef(sf, null, intfType) == null)
incList.list = new String[](incList.list, new String(intfType))
//auto-instance name ?
if (tokens[start].type == ParseToken.TYPE_PARTICLE)
{
start ++
count ++
}
//stop?
if (tokens[start].content == "{")
{
break
}
//comma
start ++
count ++
}
}
return count + skipScope(tokens, start)
}
//get the fully-qualified "package path" for which the given source file would be accessed via e.g. "uses" or "extends"
char[] getSourceFilePackage(SourceFile sf, String searchPaths[])
{
char path[] = sf.path
//check if the file was on a search path, and strip that search path if so...
for (int i = 0; i < searchPaths.arrayLength; i++)
{
if (path.startsWith(searchPaths[i].string))
{
path = path.subString(searchPaths[i].string.arrayLength+1, (path.arrayLength - searchPaths[i].string.arrayLength)-1)
break
}
}
if (path.startsWith("resources/"))
path = path.lsplit("/")[1].string
if (path.endsWith(".dn")) path = path.rsplit(".")[0].string
return path.explode("/").implode(".")
}
//here we assign a type's fully-qualified package name, relative to the file it appeared in
char[] getFullTypeName(SourceFile sf, char name[], String searchPaths[])
{
char pkg[] = getSourceFilePackage(sf, searchPaths)
String parts[] = pkg.explode(".")
if (parts[parts.arrayLength-1].string == name) name = pkg
//TODO: what should the "else" case actually be here? what does the language spec say?
return name
}
void addType(SourceFile sf, char type[], char name[], char docComment[], String searchPaths[])
{
if (type == "interface")
{
InterfaceDef nid = new InterfaceDef(TypeDef.INTERFACE, name)
nid.doc_description = docComment
nid.fullName = getFullTypeName(sf, name, searchPaths)
sf.types = new TypeDef[](sf.types, nid)
}
else if (type == "data")
{
DataDef tid = new DataDef(TypeDef.DATA, name)
tid.doc_description = docComment
tid.fullName = getFullTypeName(sf, name, searchPaths)
sf.types = new TypeDef[](sf.types, tid)
}
}
InterfaceDef getInterfaceDef(SourceFile sf, char pkg[], char name[])
{
for (int i = 0; i < sf.types.arrayLength; i++)
{
//out.println(" - check $name vs $(sf.types[i].name) / $(sf.types[i].fullName)")
if (sf.types[i].class == TypeDef.INTERFACE && (sf.types[i].name == name || sf.types[i].fullName == name))
{
InterfaceDef id = sf.types[i]
return id
}
}
for (int n = 0; n < sf.supportFiles.arrayLength; n++)
{
for (int i = 0; i < sf.supportFiles[n].types.arrayLength; i++)
{
//out.println(" - check $name vs $(sf.supportFiles[n].types[i].name) / $(sf.supportFiles[n].types[i].fullName)")
if (sf.supportFiles[n].types[i].class == TypeDef.INTERFACE && (sf.supportFiles[n].types[i].name == name || sf.supportFiles[n].types[i].fullName == name))
{
InterfaceDef id = sf.supportFiles[n].types[i]
return id
}
}
}
return null
}
DataDef getDataDef(SourceFile sf, char name[])
{
//out.println("looking for type $name")
for (int i = 0; i < sf.types.arrayLength; i++)
{
if (sf.types[i].class == TypeDef.DATA && sf.types[i].name == name)
{
DataDef dd = sf.types[i]
return dd
}
}
for (int n = 0; n < sf.supportFiles.arrayLength; n++)
{
for (int i = 0; i < sf.supportFiles[n].types.arrayLength; i++)
{
if (sf.supportFiles[n].types[i].class == TypeDef.DATA && sf.supportFiles[n].types[i].name == name)
{
DataDef dd = sf.supportFiles[n].types[i]
return dd
}
}
}
return null
}
bool isType(SourceFile sf, char name[])
{
return getInterfaceDef(sf, null, name) != null || getDataDef(sf, name) != null || isBuiltinType(name)
}
bool isPlainName(SourceFile sf, char name[])
{
return !isType(sf, name) && !isToken(name) && !isKeyword(name)
}
bool isTypeQualifier(char str[])
{
return str == "const" || str == "transfer"
}
bool isReturnTypeQualifier(char str[])
{
return false
}
bool isParamTypeQualifier(char str[])
{
return str == "store" || str == "opt"
}
bool isFieldDeclaration(SourceFile sf, ParseToken tokens[], int start)
{
//[const|transfer|copy] [sqBrackets]
while (isTypeQualifier(tokens[start].content))
start ++
if (!isType(sf, tokens[start].content))
return false
start ++
if (!isPlainName(sf, tokens[start].content))
return false
start ++
//array?
while (tokens[start].content == "[")
{
start ++
if (tokens[start].content.isNumeric())
start ++
if (tokens[start].content != "]")
return false
start ++
}
return true
}
bool isEventSourceDeclaration(SourceFile sf, ParseToken tokens[], int start)
{
//[const|transfer|copy] [sqBrackets]
if (tokens[start].content != "event")
return false
start ++
if (!isPlainName(sf, tokens[start].content))
return false
start ++
return true
}
bool isFunctionDeclaration(SourceFile sf, InterfaceDef forInterface, ParseToken tokens[], int start)
{
//regular:
//[copy] ([] [,])
//constructor:
// ([] [,])
if (tokens[start].content == forInterface.name && tokens[start+1].content == "(")
{
//constructor
start ++
}
else
{
//regular function
while (isReturnTypeQualifier(tokens[start].content))
start ++
//return type
if (!isType(sf, tokens[start].content))
{
return false
}
start ++
//array?
while (tokens[start].content == "[")
{
if (tokens[start+1].content != "]")
return false
start += 2
}
//function name
if (!isPlainName(sf, tokens[start].content))
return false
start ++
}
//parameter list
if (tokens[start].content != "(")
return false
start ++
while (start < tokens.arrayLength)
{
if (tokens[start].content == ")")
break
start ++
}
return true
}
bool isDocComment(char comment[])
{
//for now, trim the comment and check if it starts and ends with a "{" and "}"
//TODO: a full json syntax check
char tc[] = stringUtil.trim(comment)
return tc[0] == "{" && tc[tc.arrayLength-1] == "}"
}
int consumeInterfaceFieldDeclaration(InterfaceDef id, ParseToken docComment, ParseToken tokens[], int start)
{
//[const|transfer|copy] [sqBrackets]
FieldDef field = new FieldDef()
bool qTransfer
bool qConstant
int os = start
while (isTypeQualifier(tokens[start].content))
{
if (tokens[start].content == "transfer")
qTransfer = true
if (tokens[start].content == "const")
qConstant = true
start ++
}
//type name
field.type = tokens[start].content
start ++
//field name
field.name = tokens[start].content
field.displayName = field.name
start ++
//array?
while (tokens[start].content == "[")
{
start ++
if (tokens[start].content.isNumeric())
start ++
field.displayName = new char[](field.displayName, "[]")
field.array = true
start ++
}
//constant initialiser
if (qConstant)
{
if (tokens[start].content == "=")
{
start ++
//consume one expression
ParseToken tk[] = consumeExpression(tokens, start)
start += tk.arrayLength
}
}
if (docComment != null) field.doc_description = docComment.content
if (qTransfer)
id.transferFields = new FieldDef[](id.transferFields, field)
else if (qConstant)
id.constants = new FieldDef[](id.constants, field)
return start - os
}
int consumeInterfaceEventSourceDeclaration(InterfaceDef id, ParseToken docComment, ParseToken tokens[], int start)
{
//eventsource
EventSourceDef field = new EventSourceDef()
int os = start
//eventsource
//...
start ++
//field name
field.name = tokens[start].content
field.displayName = field.name
start ++
//parameter list
if (tokens[start].content != "(")
return 0
start ++
while (start < tokens.arrayLength)
{
if (tokens[start].content == ")")
break
//[modifiers] [sqBrackets]
while (isReturnTypeQualifier(tokens[start].content))
start ++
FieldDef nfd = new FieldDef()
nfd.type = tokens[start].content
start ++
nfd.name = tokens[start].content
nfd.displayName = nfd.name
start ++
if (tokens[start].content == "[")
{
if (tokens[start+1].content != "]")
return 0
start += 2
nfd.displayName = new char[](nfd.displayName, "[]")
}
field.params = new FieldDef[](field.params, nfd)
if (tokens[start].content == ",")
start ++
}
start ++
if (docComment != null) field.doc_description = docComment.content
id.eventSources = new EventSourceDef[](id.eventSources, field)
return start - os
}
int consumeInterfaceFunctionDeclaration(InterfaceDef id, ParseToken docComment, ParseToken tokens[], int start, char inSource[])
{
//regular:
// ([] [,])
//constructor:
// ([] [,])
int os = start
FunctionDef newFunction = new FunctionDef()
if (tokens[start].content == id.name && tokens[start+1].content == "(")
{
//constructor
newFunction.name = tokens[start].content
start ++
}
else
{
//regular function
while (isReturnTypeQualifier(tokens[start].content))
start ++
//return type
newFunction.returnType = tokens[start].content
start ++
//array?
while (tokens[start].content == "[")
{
if (tokens[start+1].content != "]")
return 0
start += 2
newFunction.returnType = new char[](newFunction.returnType, "[]")
}
//function name
newFunction.name = tokens[start].content
start ++
}
//parameter list
if (tokens[start].content != "(")
return 0
start ++
bool optLatch = false
while (start < tokens.arrayLength)
{
if (tokens[start].content == ")")
break
//[modifiers] [sqBrackets]
bool sStore = false
bool pOpt = optLatch
while (isParamTypeQualifier(tokens[start].content))
{
if (tokens[start].content == "store")
sStore = true
if (tokens[start].content == "opt")
{
pOpt = true
optLatch = true
}
start ++
}
FieldDef nfd = new FieldDef()
nfd.type = tokens[start].content
start ++
nfd.name = tokens[start].content
nfd.displayName = nfd.name
start ++
while (tokens[start].content == "[")
{
if (tokens[start+1].content != "]")
return 0
start += 2
nfd.displayName = new char[](nfd.displayName, "[]")
}
nfd.scope_store = sStore
nfd.opt_param = pOpt
newFunction.params = new FieldDef[](newFunction.params, nfd)
if (tokens[start].content == ",")
start ++
}
start ++
//function body?
if (tokens[start].content == "{")
{
int charStart = tokens[start].sourceStart + tokens[start].sourceLength
start ++
int depth = 1
while (depth != 0)
{
if (tokens[start].content == "}")
depth --
start ++
}
char sub[] = null
if (charStart < tokens[start-1].sourceStart - 1)
{
sub = dana.sub(inSource, charStart, tokens[start-1].sourceStart - 1)
sub = sub.explode("\r").implode("")
}
else
{
sub = " "
}
newFunction.defaultFunction = sub
}
if (docComment != null) newFunction.doc_description = docComment.content
id.functions = new FunctionDef[](id.functions, newFunction)
return start - os
}
ParseToken[] consumeExpression(ParseToken tokens[], int start)
{
// |
//we're cheating a bit: at the type level, expression can only be valid if they're constant-initialisers, which can only be a single value or "new"
if (tokens[start].content == "new")
{
ParseToken result[]
result = new ParseToken[](result, tokens[start])
start ++
//type
result = new ParseToken[](result, tokens[start])
start ++
//possible square brackets
if (tokens[start].content == "[")
{
result = new ParseToken[](result, tokens[start])
start ++
if (tokens[start].content.isNumeric())
{
result = new ParseToken[](result, tokens[start])
start ++
}
if (tokens[start].content == "]")
{
result = new ParseToken[](result, tokens[start])
start ++
}
}
//round brackets
int bb = 0
if (tokens[start].content == "(")
{
bb = 1
result = new ParseToken[](result, tokens[start])
start ++
while (bb != 0)
{
if (tokens[start].content == "(")
bb ++
if (tokens[start].content == ")")
bb --
result = new ParseToken[](result, tokens[start])
start ++
}
}
return result
}
else
{
return new ParseToken[](tokens[start])
}
}
int consumeDataFieldDeclaration(DataDef td, ParseToken docComment, ParseToken tokens[], int start)
{
//[const|transfer|copy] [sqBrackets]
FieldDef field = new FieldDef()
bool qCopy
bool qTransfer
bool qConstant
int os = start
while (isTypeQualifier(tokens[start].content))
{
if (tokens[start].content == "copy")
qCopy = true
if (tokens[start].content == "transfer")
qTransfer = true
if (tokens[start].content == "const")
qConstant = true
start ++
}
//type name
field.type = tokens[start].content
start ++
//field name
field.name = tokens[start].content
field.displayName = field.name
start ++
//array?
while (tokens[start].content == "[")
{
start ++
if (tokens[start].content.isNumeric())
start ++
field.displayName = new char[](field.displayName, "[]")
field.array = true
start ++
}
//constant initialiser
if (qConstant)
{
if (tokens[start].content == "=")
{
start ++
//consume one expression
ParseToken tk[] = consumeExpression(tokens, start)
start += tk.arrayLength
}
}
if (docComment != null) field.doc_description = docComment.content
if (qConstant)
td.constants = new FieldDef[](td.constants, field)
else
td.fields = new FieldDef[](td.fields, field)
return start - os
}
void printTypesFor(SourceFile sf)
{
out.println("types in '$(sf.path)'")
for (int i = 0; i < sf.types.arrayLength; i++)
{
out.println(" -- $(sf.types[i].name)")
}
}
int parseTypeDefinition(ParseStatus status, SourceFile root_sf, SourceFile sf, ParseToken tokens[], int start, char inSource[])
{
//collect the type name and also extract any "extends" clause
//out.println("[parse type $(tokens[start+1].content)]")
int count = 0
if (tokens[start].content == "interface")
{
InterfaceDef id = getInterfaceDef(root_sf, null, tokens[start+1].content)
if (id == null)
{
throw new Exception("Interface $(tokens[start+1].content) not found!!!")
}
//type keyword
start ++
count ++
//type name
start ++
count ++
//extends?
if (tokens[start].content == "extends")
{
start ++
count ++
//(package) and type name of the thing we're extending
char pkg[]
char nm[] = tokens[start].content
pkg = new char[](pkg, nm)
start ++
count ++
while (tokens[start].content == ".")
{
start ++
count ++
nm = tokens[start].content
pkg = new char[](pkg, ".", nm)
start ++
count ++
}
id.extendsType = getInterfaceDef(root_sf, pkg, nm)
}
//the type definition
if (tokens[start].content == "{")
{
count ++
start ++
ParseToken lastDocComment
while (start < tokens.arrayLength)
{
int ac = 1
if (isFunctionDeclaration(root_sf, id, tokens, start))
{
//out.println(" - function")
ac = consumeInterfaceFunctionDeclaration(id, lastDocComment, tokens, start, inSource)
lastDocComment = null
}
else if (isFieldDeclaration(root_sf, tokens, start))
{
//out.println(" - field")
ac = consumeInterfaceFieldDeclaration(id, lastDocComment, tokens, start)
lastDocComment = null
}
else if (isEventSourceDeclaration(root_sf, tokens, start))
{
//out.println(" - eventsource")
ac = consumeInterfaceEventSourceDeclaration(id, lastDocComment, tokens, start)
lastDocComment = null
}
else if (tokens[start].content == "}")
{
break
}
else if (tokens[start].type == ParseToken.TYPE_BLOCK_COMMENT || tokens[start].type == ParseToken.TYPE_LINE_COMMENT)
{
//out.println(" - comment")
if (isDocComment(tokens[start].content))
lastDocComment = tokens[start]
}
else
{
status.issues = new ParseIssue[](status.issues, new ParseIssue(ParseIssue.I_SYNTAX_ERROR, id.name, sf.path, "unknown syntax at $(tokens[start].content)"))
return 0
}
count += ac
start += ac
//out.println(" - next token: $(tokens[start].content)")
}
}
}
else if (tokens[start].content == "data")
{
DataDef td = getDataDef(sf, tokens[start+1].content)
//type keyword
start ++
count ++
//type name
start ++
count ++
//nocycle?
if (tokens[start].content == "nocycle")
{
start ++
count ++
}
//extends?
if (tokens[start].content == "extends")
{
start ++
count ++
//(package) and type name of the thing we're extending
char pkg[]
pkg = new char[](pkg, tokens[start].content)
start ++
count ++
while (tokens[start].content == ".")
{
start ++
count ++
pkg = new char[](pkg, ".", tokens[start].content)
start ++
count ++
}
td.extendsType = getDataDef(root_sf, pkg)
}
//nocycle?
if (tokens[start].content == "nocycle")
{
start ++
count ++
}
//the type definition
if (tokens[start].content == "{")
{
start ++
count ++
ParseToken lastDocComment
while (start < tokens.arrayLength)
{
int ac = 1
if (isFieldDeclaration(root_sf, tokens, start))
{
//out.println(" - field")
ac = consumeDataFieldDeclaration(td, lastDocComment, tokens, start)
lastDocComment = null
}
else if (tokens[start].content == "}")
{
break
}
else if (tokens[start].type == ParseToken.TYPE_BLOCK_COMMENT || tokens[start].type == ParseToken.TYPE_LINE_COMMENT)
{
//out.println(" - comment")
if (isDocComment(tokens[start].content))
lastDocComment = tokens[start]
}
else
{
status.issues = new ParseIssue[](status.issues, new ParseIssue(ParseIssue.I_SYNTAX_ERROR, td.name, sf.path, "unknown syntax at $(tokens[start].content)"))
return 0
}
count += ac
start += ac
//out.println(" - next token: $(tokens[start].content)")
}
}
}
return count
}
void parseTypeDetails(SourceFile root_sf, SourceFile sf, ParseToken tokens[], ParseStatus status, char inSource[])
{
//in this pass we have collected all valid type names, so we can do a full syntax check (group into expressions, type definitions etc.)
int start = 0
while (start != tokens.arrayLength)
{
int clength = 1
if (isTypeDefinition(tokens, start))
{
clength = parseTypeDefinition(status, root_sf, sf, tokens, start, inSource)
}
if (clength == 0) return
start += clength
}
}
SourceFile findSourceFile(char path[])
{
for (int i = 0; i < primaryFiles.arrayLength; i++)
{
if (primaryFiles[i].path == path)
return primaryFiles[i]
}
for (int i = 0; i < supportFiles.arrayLength; i++)
{
if (supportFiles[i].path == path)
return supportFiles[i]
}
return null
}
char[] getRelativePath(char path[], String searchPaths[])
{
//check if the file was on a search path, and strip that search path if so...
for (int i = 0; i < searchPaths.arrayLength; i++)
{
if (path.startsWith(searchPaths[i].string))
{
path = path.subString(searchPaths[i].string.arrayLength, (path.arrayLength - searchPaths[i].string.arrayLength))
break
}
}
return path
}
char[] resolvePackageToFile(SourceFile sf, char package[], String searchPaths[], KeyValue resFiles[])
{
char epath[] = new char[](stringUtil.implode(stringUtil.explode(package, "."), "/"), ".dn")
char test[]
// - check relative to the source file that references it
char rqpath[] = getRelativePath(sf.path, searchPaths)
String tokens[] = clone stringUtil.explode(rqpath, "/\\")
//out.println("epath: $epath")
//out.println("rqpath: $rqpath")
if (tokens[0].string == "resources") tokens = dana.sub(tokens, 1, tokens.arrayLength-1)
if (tokens.arrayLength > 1)
tokens = dana.sub(tokens, 0, tokens.arrayLength-2)
else
tokens = null
if (tokens.arrayLength > 0)
test = new char[](stringUtil.implode(tokens, "/"), "/", epath)
else
test = epath
char simplePath[] = epath
char relativePath[] = test
test = "resources/$simplePath"
//out.println(" - scan $test")
if (fs.exists("resources/$simplePath"))
return test
test = "resources/$relativePath"
//out.println(" - scan $test")
if (fs.exists("resources/$relativePath"))
return test
// - check in-memory files via resFiles
for (int i = 0; i < resFiles.arrayLength; i++)
{
test = "resources/$simplePath"
//out.println(" - scan $test")
if (resFiles[i].key == test)
return test
test = "resources/$relativePath"
//out.println(" - scan $test")
if (resFiles[i].key == test)
return test
}
// - check search paths
for (int i = 0; i < searchPaths.arrayLength; i++)
{
test = "$(searchPaths[i].string)/resources/$(simplePath)"
//out.println(" - scan $test")
if (fs.exists(test))
return test
test = "$(searchPaths[i].string)/resources/$(relativePath)"
//out.println(" - scan $test")
if (fs.exists(test))
return test
}
return null
}
bool hasSupportFile(SourceFile sf, SourceFile q)
{
for (int i = 0; i < sf.supportFiles.arrayLength; i++)
{
if (sf.supportFiles[i] === q)
return true
}
return false
}
void addSubSupportFiles(SourceFile to_sf, SourceFile from_sf)
{
for (int i = 0; i < from_sf.supportFiles.arrayLength; i++)
{
if (! hasSupportFile(to_sf, from_sf.supportFiles[i]))
{
to_sf.supportFiles = new SourceFile[](to_sf.supportFiles, from_sf.supportFiles[i])
addSubSupportFiles(to_sf, from_sf.supportFiles[i])
}
}
}
void removeIssue(ParseStatus status, int index)
{
ParseIssue ni[] = new ParseIssue[status.issues.arrayLength - 1]
int j = 0
for (int i = 0; i < status.issues.arrayLength; i++)
{
if (i == index)
{
}
else
{
ni[j] = status.issues[i]
j ++
}
}
status.issues = ni
}
void resolveTypeLookups(SourceFile sf, ParseStatus status)
{
for (int i = 0; i < sf.types.arrayLength; i++)
{
for (int j = 0; j < status.issues.arrayLength; j++)
{
if (status.issues[j].type == ParseIssue.I_UNRESOLVED_TYPE && status.issues[j].element == sf.types[i].name)
{
removeIssue(status, j)
break
}
}
}
}
void collectExternalTypes(SourceFile root_sf, SourceFile sf, IncludeList incList, ParseStatus status, String searchPaths[], KeyValue resFiles[])
{
//scan over all "uses" entries and "extends" clauses, but don't visit any files already in parsedFiles
for (int i = 0; i < incList.list.arrayLength; i++)
{
char package[] = incList.list[i].string
//resolve this package into a file to parse, relative to sf.path
char path[] = resolvePackageToFile(sf, package, searchPaths, resFiles)
//out.println("incList: $package / $path")
if (path == null)
{
//check if this type name has since been located in another file
//TODO: the incList needs more info on where this inc came from (at least whether it's expecting a data type or interface!!)
if (getDataDef(sf, package) == null && getInterfaceDef(sf, null, package) == null)
{
status.issues = new ParseIssue[](status.issues, new ParseIssue(ParseIssue.I_UNRESOLVED_TYPE, package, sf.path, "unresolved type '$package'"))
}
}
else
{
//check if this file is already in primaryFiles or supportFiles
// - if not, go ahead and call parseTokens on it
SourceFile q = findSourceFile(path)
if (q == null)
{
char content[] = null
KeyValue memStaged = null
if ((memStaged = resFiles.findFirst(KeyValue.[key], new KeyValue(path))) != null)
{
content = memStaged.value
}
else
{
File fd = new File(path, File.READ)
content = fd.read(fd.getSize())
fd.close()
}
if (content != null)
{
q = new SourceFile(path)
q.package = getSourceFilePackage(q, searchPaths)
supportFiles = new SourceFile[](supportFiles, q)
if (root_sf !== sf) sf.supportFiles = new SourceFile[](sf.supportFiles, q)
root_sf.supportFiles = new SourceFile[](root_sf.supportFiles, q)
ParseToken tokens[] = tokeniser.tokenise(content).tokens
parseTokens(root_sf, q, tokens, status, searchPaths, resFiles, content)
//scan the types added in 'q', and check if we can now remove any parse issues for unfindable types
resolveTypeLookups(q, status)
}
}
else
{
if (!hasSupportFile(root_sf, q))
{
root_sf.supportFiles = new SourceFile[](root_sf.supportFiles, q)
//we still need to add all supportFiles that q itself uses to root_sf's supportFiles
addSubSupportFiles(root_sf, q)
}
if (sf !== q && !hasSupportFile(sf, q))
{
sf.supportFiles = new SourceFile[](sf.supportFiles, q)
addSubSupportFiles(sf, q)
}
}
}
}
}
void parseTokens(SourceFile root_sf, SourceFile sf, ParseToken tokens[], ParseStatus status, String searchPaths[], KeyValue resFiles[], char inSource[])
{
//group tokens into expressions, type definitions etc.
// - we need to do the same multi-pass process as the compiler: first scan for type definitions, then populate the types with their fields, etc.
int start = 0
IncludeList incList = new IncludeList()
char lastDocComment[]
while (start != tokens.arrayLength)
{
int clength = 1
if (isUses(tokens, start))
{
clength = consumeUses(tokens, start, incList)
}
else if (isTypeDefinition(tokens, start))
{
clength = consumeTypeDefinition(sf, lastDocComment, tokens, start, incList, searchPaths)
lastDocComment = null
}
else if (isComponentDefinition(tokens, start))
{
clength = consumeComponentDefinition(sf, lastDocComment, tokens, start, incList)
lastDocComment = null
}
else if (tokens[start].type == ParseToken.TYPE_BLOCK_COMMENT || tokens[start].type == ParseToken.TYPE_LINE_COMMENT)
{
if (isDocComment(tokens[start].content))
lastDocComment = tokens[start].content
}
start += clength
}
//from the headers of "uses" and type definitions, collect all external types from other source files
// - the below function should check supportFiles first to see if already parsed, and if so just add a reference to sf's list
collectExternalTypes(root_sf, sf, incList, status, searchPaths, resFiles)
//now that we have all externally-defined types, parse the internals of the type definitions (i.e. fields) from this source file
parseTypeDetails(root_sf, sf, tokens, status, inSource)
}
TypeParser:TypeParser()
{
initialiseParseTokens()
initialiseParseKeywords()
initialiseBuiltinTypes()
tokeniser = new Tokeniser(parseTokens)
tokeniser.setLineComment("//")
tokeniser.setBlockComment("/*", "*/")
}
void addAutoInterfaces(SourceFile sf, ParseStatus status, String searchPaths[], KeyValue resFiles[])
{
// for any components found, if they provide an interface which is a sub-type, we need a secondary interface of type AdaptEvents (this matches what the compiler does) -- so if no such secondary interface is found, add one
for (int i = 0; i < sf.types.arrayLength; i++)
{
if (sf.types[i].class == TypeDef.COMPONENT)
{
ComponentDef cdef = sf.types[i]
for (int j = 0; j < cdef.provIntfs.arrayLength; j++)
{
InterfaceDef id = getInterfaceDef(sf, null, cdef.provIntfs[j].typeName)
if (id != null && id.extendsType != null && id.extendsType.fullName != "lang.Object")
{
if (cdef.provIntfs[j].subInterfaces.findFirst(String.[string], new String("AdaptEvents")) == null)
{
//out.println(" -- adding auto interface AdaptEvents")
cdef.provIntfs[j].subInterfaces = new String[](cdef.provIntfs[j].subInterfaces, new String("AdaptEvents"))
collectExternalTypes(sf, sf, new IncludeList(new String[](new String("AdaptEvents"))), status, searchPaths, resFiles)
}
}
}
}
}
}
ParseStatus TypeParser:parseFile(char content[], opt char filePath[], opt String searchPaths[], opt KeyValue resFiles[])
{
ParseStatus status = new ParseStatus(true)
TokeniseResult tr = tokeniser.tokenise(content)
if (tr.issues.arrayLength != 0)
{
status.success = false
status.issues = new ParseIssue(ParseIssue.I_PARSE_FAILURE, filePath, filePath, tr.issues[0].description)
return status
}
ParseToken tokens[] = tr.tokens
//out.println("[[beginning parse for '$filePath']]")
/*
for (int i = 0; i < tokens.arrayLength; i++)
{
out.println("<$(iu.makeString(tokens[i].type))>token: $(tokens[i].content)")
}
*/
if (!(isset filePath) || filePath == null) filePath = ""
if (!(isset searchPaths)) searchPaths = new String[](new String("$(sysInfo.getDanaHome())/components/"))
SourceFile sf = new SourceFile(filePath)
sf.package = getSourceFilePackage(sf, searchPaths)
IncludeList incList = new IncludeList()
incList.list = new String[](incList.list, new String("lang.Type"))
incList.list = new String[](incList.list, new String("lang.Object"))
incList.list = new String[](incList.list, new String("lang.IDC"))
incList.list = new String[](incList.list, new String("lang.Thread"))
incList.list = new String[](incList.list, new String("lang.Destructor"))
incList.list = new String[](incList.list, new String("lang.AdaptEvents"))
collectExternalTypes(sf, sf, incList, status, searchPaths, resFiles)
parseTokens(sf, sf, tokens, status, searchPaths, resFiles, content)
//add any interfaces that are injected by the Dana compiler
addAutoInterfaces(sf, status, searchPaths, resFiles)
primaryFiles = new SourceFile[](primaryFiles, sf)
return status
}
ParsedFiles TypeParser:getParsedFiles()
{
return new ParsedFiles(primaryFiles, supportFiles)
}
}
Revision history
To propose a new revision to this entity, use
dana source put -uc your/new/version.dn -n parsing.TypeParser -m "reason for update" -u yourUsernameVersion 2 (this version) by barry
Notes for this version: Adds logic to extract default function implementations.
Version 1 by barry