Sunday, January 30, 2011

ANTLR - Java source code rewrite

See ANTLR4 for other solution using Listener
https://theantlrguy.atlassian.net/wiki/display/ANTLR4/Parse+Tree+Listeners
http://media.pragprog.com/titles/tpantlr2/listener.pdf

Using examples from chapter 9 in http://www.pragprog.com/titles/tpantlr/the-definitive-antlr-reference

Java tree grammar used for rewrite example
http://www.antlr.org/grammar/1207932239307/Java1_5Grammars
Dieter Habelitz - A Java 1.5 grammar that builds trees and a tree grammar. Not derived from Java grammar Terence wrote.
original link broken - http://www.antlr3.org/grammar/1207932239307/Java1_5Grammars

Try this on  github
https://github.com/grammarware/slps/tree/master/topics/grammars/java-5/habelitz 

Add template rewrite call into tree grammar JavaTreeParser.g
  |   ^(METHOD_CALL  primaryExpression {$primaryExpression::isMethodCall=true;} genericTypeArgumentList? arguments)
      -> methodCall(methodCallText={$text})


Create template file JavaTreeTemplate.stg
group Java;

methodCall(methodCallText) ::= <<
$methodCallText$;
log("Called $methodCallText$")
>>


import java.io.FileReader;

import generated.JavaLexer;
import generated.JavaParser;
import generated.JavaTreeParser;

import org.antlr.runtime.ANTLRFileStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.TokenRewriteStream;
import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.stringtemplate.StringTemplateGroup;
import org.antlr.stringtemplate.language.AngleBracketTemplateLexer;
import org.antlr.stringtemplate.language.DefaultTemplateLexer;


public class JavaTreeWalker {
    
    public static void main(String[] args) throws Exception {
        
        String templateFileName = "src/JavaTreeTemplate.stg";
        StringTemplateGroup templates = new StringTemplateGroup(new FileReader(templateFileName),
                            DefaultTemplateLexer.class);
        
        ANTLRFileStream input = new ANTLRFileStream(args[0]);
        
        JavaLexer lexer = new JavaLexer(input);
        
        TokenRewriteStream tokens = new TokenRewriteStream(lexer);
        
        JavaParser parser = new JavaParser(tokens);
        
        JavaParser.javaSource_return r = parser.javaSource();
        
        CommonTree t = (CommonTree)r.getTree();
        
        System.out.println(t.toStringTree());
        
        CommonTreeNodeStream nodes = new CommonTreeNodeStream(t);
        
        nodes.setTokenStream(tokens);
        
        JavaTreeParser walker = new JavaTreeParser(nodes);
        
        walker.setTemplateLib(templates);
        
        System.out.println("\nWalk tree:\n");
        printTree(t,0);
        
        walker.javaSource();
        
        System.out.println(tokens.toString());
        
    }
    
    public static void printTree(CommonTree t, int indent) {
        if ( t != null ) {
            StringBuffer sb = new StringBuffer(indent);
            for ( int i = 0; i < indent; i++ )
                sb = sb.append("   ");
            for ( int i = 0; i < t.getChildCount(); i++ ) {
                System.out.println(sb.toString() + t.getChild(i).toString());
                printTree((CommonTree)t.getChild(i), indent+1);
            }
        }
    }


}


Original HelloWorld Java program
public class HelloWorld {

    static int x;

    /**
     * This is test class for ANTLR Java source code transformation
     * @param args
     */
    public static void main(String args[]) {

        int x;
        int y;
        
        x = 1; 
        print(x);

        System.out.println("Hello,world!");

    }
    
    private static  void print(int x){
        System.out.println(x);
    }

}


Program after rewrite by ANTLR
public class HelloWorld {

    static int x;

    /**
     * This is test class for ANTLR Java source code transformation
     * @param args
     */
    public static void main(String args[]) {

        int x;
        int y;
        
        x = 1; 
        print(x);
log("Called print(x)");

        System.out.println("Hello,world!");
log("Called System.out.println("Hello,world!")");

    }
    
    private static  void print(int x){
        System.out.println(x);
log("Called System.out.println(x)");
    }

}


ANTLR AST
(JAVA_SOURCE ANNOTATION_LIST (class (MODIFIER_LIST public) HelloWorld (CLASS_TOP_LEVEL_SCOPE (VAR_DECLARATION (MODIFIER_LIST static) (TYPE int) (VAR_DECLARATOR_LIST (VAR_DECLARATOR x))) (VOID_METHOD_DECL (MODIFIER_LIST public static) main (FORMAL_PARAM_LIST (FORMAL_PARAM_STD_DECL LOCAL_MODIFIER_LIST (TYPE (QUALIFIED_TYPE_IDENT String)) (args (ARRAY_DECLARATOR_LIST ARRAY_DECLARATOR)))) (BLOCK_SCOPE (VAR_DECLARATION LOCAL_MODIFIER_LIST (TYPE int) (VAR_DECLARATOR_LIST (VAR_DECLARATOR x))) (VAR_DECLARATION LOCAL_MODIFIER_LIST (TYPE int) (VAR_DECLARATOR_LIST (VAR_DECLARATOR y))) (EXPR (= x 1)) (EXPR (METHOD_CALL print (ARGUMENT_LIST (EXPR x)))) (EXPR (METHOD_CALL (. (. System out) println) (ARGUMENT_LIST (EXPR "Hello,world!")))))) (VOID_METHOD_DECL (MODIFIER_LIST private static) print (FORMAL_PARAM_LIST (FORMAL_PARAM_STD_DECL LOCAL_MODIFIER_LIST (TYPE int) x)) (BLOCK_SCOPE (EXPR (METHOD_CALL (. (. System out) println) (ARGUMENT_LIST (EXPR x)))))))))

Walk tree:

ANNOTATION_LIST
class
   MODIFIER_LIST
      public
   HelloWorld
   CLASS_TOP_LEVEL_SCOPE
      VAR_DECLARATION
         MODIFIER_LIST
            static
         TYPE
            int
         VAR_DECLARATOR_LIST
            VAR_DECLARATOR
               x
      VOID_METHOD_DECL
         MODIFIER_LIST
            public
            static
         main
         FORMAL_PARAM_LIST
            FORMAL_PARAM_STD_DECL
               LOCAL_MODIFIER_LIST
               TYPE
                  QUALIFIED_TYPE_IDENT
                     String
               args
                  ARRAY_DECLARATOR_LIST
                     ARRAY_DECLARATOR
         BLOCK_SCOPE
            VAR_DECLARATION
               LOCAL_MODIFIER_LIST
               TYPE
                  int
               VAR_DECLARATOR_LIST
                  VAR_DECLARATOR
                     x
            VAR_DECLARATION
               LOCAL_MODIFIER_LIST
               TYPE
                  int
               VAR_DECLARATOR_LIST
                  VAR_DECLARATOR
                     y
            EXPR
               =
                  x
                  1
            EXPR
               METHOD_CALL
                  print
                  ARGUMENT_LIST
                     EXPR
                        x
            EXPR
               METHOD_CALL
                  .
                     .
                        System
                        out
                     println
                  ARGUMENT_LIST
                     EXPR
                        "Hello,world!"
      VOID_METHOD_DECL
         MODIFIER_LIST
            private
            static
         print
         FORMAL_PARAM_LIST
            FORMAL_PARAM_STD_DECL
               LOCAL_MODIFIER_LIST
               TYPE
                  int
               x
         BLOCK_SCOPE
            EXPR
               METHOD_CALL
                  .
                     .
                        System
                        out
                     println
                  ARGUMENT_LIST
                     EXPR
                        x

Saturday, January 29, 2011

ANTLR - Java source tree walker

Java 1.5 grammar for ANTLR v3 that builds trees
 
Dieter Habelitz - A Java 1.5 grammar that builds trees and a tree grammar. Not derived from Java grammar Terence wrote.
original link broken - http://www.antlr3.org/grammar/1207932239307/Java1_5Grammars

Try this on  github
https://github.com/grammarware/slps/tree/master/topics/grammars/java-5/habelitz

Compile grammars
java org.antlr.Tool -o generated Java.g
java org.antlr.Tool -o generated JavaTreeParser.g


Creates following files:
 ls -l generated/
total 1588
-rw-------. 1 dave vesedan  145133 Jan 29 12:20 JavaLexer.java
-rw-------. 1 dave vesedan 1046974 Jan 29 12:20 JavaParser.java
-rw-r--r--. 1 dave vesedan    3687 Jan 29 12:10 Java.tokens
-rw-r--r--. 1 dave vesedan  386412 Jan 29 13:11 JavaTreeParser.java
-rw-r--r--. 1 dave vesedan    3687 Jan 29 13:11 JavaTreeParser.tokens


import generated.JavaLexer;
import generated.JavaParser;
import generated.JavaTreeParser;

import org.antlr.runtime.ANTLRFileStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.CommonTreeNodeStream;


public class JavaTreeWalker {
    
    public static void main(String[] args) throws Exception {
        
        ANTLRFileStream input = new ANTLRFileStream(args[0]);
        
        JavaLexer lexer = new JavaLexer(input);
        
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        
        JavaParser parser = new JavaParser(tokens);
        
        JavaParser.javaSource_return r = parser.javaSource();
        
        CommonTree t = (CommonTree)r.getTree();
        
        System.out.println(t.toStringTree());
        
        CommonTreeNodeStream nodes = new CommonTreeNodeStream(t);
        
        nodes.setTokenStream(tokens);
        
        JavaTreeParser walker = new JavaTreeParser(nodes);
        
        System.out.println("\nWalk tree:\n");
        
        walker.javaSource();
        
        System.out.println(tokens.toString());
        
    }

}


(JAVA_SOURCE ANNOTATION_LIST (import (. generated JavaLexer)) (import (. generated JavaParser)) (import (. generated JavaTreeParser)) (import (. (. (. org antlr) runtime) ANTLRFileStream)) (import (. (. (. org antlr) runtime) CommonTokenStream)) (import (. (. (. (. org antlr) runtime) tree) CommonTree)) (import (. (. (. (. org antlr) runtime) tree) CommonTreeNodeStream)) (class (MODIFIER_LIST public) JavaTreeWalker (CLASS_TOP_LEVEL_SCOPE (VOID_METHOD_DECL (MODIFIER_LIST public static) main (FORMAL_PARAM_LIST (FORMAL_PARAM_STD_DECL LOCAL_MODIFIER_LIST (TYPE (QUALIFIED_TYPE_IDENT String) (ARRAY_DECLARATOR_LIST ARRAY_DECLARATOR)) args)) (THROWS_CLAUSE Exception) (BLOCK_SCOPE (VAR_DECLARATION LOCAL_MODIFIER_LIST (TYPE (QUALIFIED_TYPE_IDENT ANTLRFileStream)) (VAR_DECLARATOR_LIST (VAR_DECLARATOR input (EXPR (STATIC_ARRAY_CREATOR (QUALIFIED_TYPE_IDENT ANTLRFileStream) (ARGUMENT_LIST (EXPR (ARRAY_ELEMENT_ACCESS args (EXPR 0))))))))) (VAR_DECLARATION LOCAL_MODIFIER_LIST (TYPE (QUALIFIED_TYPE_IDENT JavaLexer)) (VAR_DECLARATOR_LIST (VAR_DECLARATOR lexer (EXPR (STATIC_ARRAY_CREATOR (QUALIFIED_TYPE_IDENT JavaLexer) (ARGUMENT_LIST (EXPR input))))))) (VAR_DECLARATION LOCAL_MODIFIER_LIST (TYPE (QUALIFIED_TYPE_IDENT CommonTokenStream)) (VAR_DECLARATOR_LIST (VAR_DECLARATOR tokens (EXPR (STATIC_ARRAY_CREATOR (QUALIFIED_TYPE_IDENT CommonTokenStream) (ARGUMENT_LIST (EXPR lexer))))))) (VAR_DECLARATION LOCAL_MODIFIER_LIST (TYPE (QUALIFIED_TYPE_IDENT JavaParser)) (VAR_DECLARATOR_LIST (VAR_DECLARATOR parser (EXPR (STATIC_ARRAY_CREATOR (QUALIFIED_TYPE_IDENT JavaParser) (ARGUMENT_LIST (EXPR tokens))))))) (VAR_DECLARATION LOCAL_MODIFIER_LIST (TYPE (QUALIFIED_TYPE_IDENT JavaParser javaSource_return)) (VAR_DECLARATOR_LIST (VAR_DECLARATOR r (EXPR (METHOD_CALL (. parser javaSource) ARGUMENT_LIST))))) (VAR_DECLARATION LOCAL_MODIFIER_LIST (TYPE (QUALIFIED_TYPE_IDENT CommonTree)) (VAR_DECLARATOR_LIST (VAR_DECLARATOR t (EXPR (CAST_EXPR (TYPE (QUALIFIED_TYPE_IDENT CommonTree)) (METHOD_CALL (. r getTree) ARGUMENT_LIST)))))) (EXPR (METHOD_CALL (. (. System out) println) (ARGUMENT_LIST (EXPR (METHOD_CALL (. t toStringTree) ARGUMENT_LIST))))) (VAR_DECLARATION LOCAL_MODIFIER_LIST (TYPE (QUALIFIED_TYPE_IDENT CommonTreeNodeStream)) (VAR_DECLARATOR_LIST (VAR_DECLARATOR nodes (EXPR (STATIC_ARRAY_CREATOR (QUALIFIED_TYPE_IDENT CommonTreeNodeStream) (ARGUMENT_LIST (EXPR t))))))) (EXPR (METHOD_CALL (. nodes setTokenStream) (ARGUMENT_LIST (EXPR tokens)))) (VAR_DECLARATION LOCAL_MODIFIER_LIST (TYPE (QUALIFIED_TYPE_IDENT JavaTreeParser)) (VAR_DECLARATOR_LIST (VAR_DECLARATOR walker (EXPR (STATIC_ARRAY_CREATOR (QUALIFIED_TYPE_IDENT JavaTreeParser) (ARGUMENT_LIST (EXPR nodes))))))) (EXPR (METHOD_CALL (. (. System out) println) (ARGUMENT_LIST (EXPR "\nWalk tree:\n")))) (EXPR (METHOD_CALL (. walker javaSource) ARGUMENT_LIST)) (EXPR (METHOD_CALL (. (. System out) println) (ARGUMENT_LIST (EXPR (METHOD_CALL (. tokens toString) ARGUMENT_LIST))))))))))

Walk tree:

importgenerated.JavaLexer;importgenerated.JavaParser;importgenerated.JavaTreeParser;importorg.antlr.runtime.ANTLRFileStream;importorg.antlr.runtime.CommonTokenStream;importorg.antlr.runtime.tree.CommonTree;importorg.antlr.runtime.tree.CommonTreeNodeStream;publicclassJavaTreeWalker{publicstaticvoidmain(String[]args)throwsException{ANTLRFileStreaminput=newANTLRFileStream(args[0]);JavaLexerlexer=newJavaLexer(input);CommonTokenStreamtokens=newCommonTokenStream(lexer);JavaParserparser=newJavaParser(tokens);JavaParser.javaSource_returnr=parser.javaSource();CommonTreet=(CommonTree)r.getTree();System.out.println(t.toStringTree());CommonTreeNodeStreamnodes=newCommonTreeNodeStream(t);nodes.setTokenStream(tokens);JavaTreeParserwalker=newJavaTreeParser(nodes);System.out.println("\nWalk tree:\n");walker.javaSource();System.out.println(tokens.toString());}}


HelloWorld Java program
public class HelloWord {

  static int x;

  public static void main(String args[]){
    
    int y;

    System.out.println("Hello,world!");

  }

}


AST text output

(JAVA_SOURCE ANNOTATION_LIST (class (MODIFIER_LIST public) HelloWorld (CLASS_TOP_LEVEL_SCOPE (VAR_DECLARATION (MODIFIER_LIST static) (TYPE int) (VAR_DECLARATOR_LIST (VAR_DECLARATOR x))) (VOID_METHOD_DECL (MODIFIER_LIST public static) main (FORMAL_PARAM_LIST (FORMAL_PARAM_STD_DECL LOCAL_MODIFIER_LIST (TYPE (QUALIFIED_TYPE_IDENT String)) (args (ARRAY_DECLARATOR_LIST ARRAY_DECLARATOR)))) (BLOCK_SCOPE (VAR_DECLARATION LOCAL_MODIFIER_LIST (TYPE int) (VAR_DECLARATOR_LIST (VAR_DECLARATOR y))) (EXPR (METHOD_CALL (. (. System out) println) (ARGUMENT_LIST (EXPR "Hello,world!")))))))))

Walk tree:

publicclassHelloWorld{staticintx;publicstaticvoidmain(Stringargs[]){inty;System.out.println("Hello,world!");}}


With option to keep whitespace set in Java.g
public boolean preserveWhitespacesAndComments = true;


(JAVA_SOURCE ANNOTATION_LIST (class (MODIFIER_LIST public) HelloWorld (CLASS_TOP_LEVEL_SCOPE (VAR_DECLARATION (MODIFIER_LIST static) (TYPE int) (VAR_DECLARATOR_LIST (VAR_DECLARATOR x))) (VOID_METHOD_DECL (MODIFIER_LIST public static) main (FORMAL_PARAM_LIST (FORMAL_PARAM_STD_DECL LOCAL_MODIFIER_LIST (TYPE (QUALIFIED_TYPE_IDENT String)) (args (ARRAY_DECLARATOR_LIST ARRAY_DECLARATOR)))) (BLOCK_SCOPE (VAR_DECLARATION LOCAL_MODIFIER_LIST (TYPE int) (VAR_DECLARATOR_LIST (VAR_DECLARATOR y))) (EXPR (METHOD_CALL (. (. System out) println) (ARGUMENT_LIST (EXPR "Hello,world!")))))))))

Walk tree:

public class HelloWorld {

  static int x;

  public static void main(String args[]){
    
    int y;

    System.out.println("Hello,world!");

  }

}




AST generated in ANTLR-Works