View Javadoc

1   /**
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   package net.sourceforge.pmd.cpd;
5   
6   import static org.junit.Assert.assertEquals;
7   import net.sourceforge.pmd.PMD;
8   
9   import org.junit.Test;
10  
11  public class JavaTokensTokenizerTest {
12  
13      @Test
14      public void test1() throws Throwable {
15          Tokenizer tokenizer = new JavaTokenizer();
16          SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("public class Foo {}"));
17          Tokens tokens = new Tokens();
18          tokenizer.tokenize(sourceCode, tokens);
19          assertEquals(6, tokens.size());
20          assertEquals("public class Foo {}", sourceCode.getSlice(1, 1));
21      }
22  
23      @Test
24      public void testCommentsIgnored() throws Throwable {
25          Tokenizer tokenizer = new JavaTokenizer();
26          SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("public class Foo { // class Bar */ \n }"));
27          Tokens tokens = new Tokens();
28          tokenizer.tokenize(sourceCode, tokens);
29          assertEquals(6, tokens.size());
30      }
31  
32      @Test
33      public void test2() throws Throwable {
34          Tokenizer t = new JavaTokenizer();
35          String data = "public class Foo {" + PMD.EOL + "public void bar() {}" + PMD.EOL + "public void buz() {}" + PMD.EOL + "}";
36          SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader(data));
37          Tokens tokens = new Tokens();
38          t.tokenize(sourceCode, tokens);
39          assertEquals("public class Foo {" + PMD.EOL + "public void bar() {}", sourceCode.getSlice(1, 2));
40      }
41  
42      @Test
43      public void testDiscardSemicolons() throws Throwable {
44          Tokenizer t = new JavaTokenizer();
45          SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("public class Foo {private int x;}"));
46          Tokens tokens = new Tokens();
47          t.tokenize(sourceCode, tokens);
48          assertEquals(9, tokens.size());
49      }
50  
51      @Test
52      public void testDiscardImports() throws Throwable {
53          Tokenizer t = new JavaTokenizer();
54          SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("import java.io.File;" + PMD.EOL + "public class Foo {}"));
55          Tokens tokens = new Tokens();
56          t.tokenize(sourceCode, tokens);
57          assertEquals(6, tokens.size());
58      }
59  
60      @Test
61      public void testDiscardPkgStmts() throws Throwable {
62          Tokenizer t = new JavaTokenizer();
63          SourceCode sourceCode = new SourceCode(new SourceCode.StringCodeLoader("package foo.bar.baz;" + PMD.EOL + "public class Foo {}"));
64          Tokens tokens = new Tokens();
65          t.tokenize(sourceCode, tokens);
66          assertEquals(6, tokens.size());
67      }
68  
69      @Test
70      public void testDiscardSimpleOneLineAnnotation() throws Throwable {
71          JavaTokenizer t = new JavaTokenizer();
72          t.setIgnoreAnnotations(true);
73          SourceCode sourceCode = new SourceCode(
74                  new SourceCode.StringCodeLoader(
75                      "package foo.bar.baz;" +
76                      PMD.EOL +
77                      "@MyAnnotation" +
78                      PMD.EOL +
79                      "public class Foo {}"
80                  ));
81          Tokens tokens = new Tokens();
82          t.tokenize(sourceCode, tokens);
83          assertEquals(6, tokens.size());
84      }
85  
86      /**
87       * Comments are discarded already by the Java parser.
88       * It would be nice, however, to use simple comments like
89       * //CPD-START or //CPD-END
90       * to enable discard-mode of CPD
91       */
92      @Test
93      public void testIgnoreComments() {
94          JavaTokenizer t = new JavaTokenizer();
95          t.setIgnoreAnnotations(false);
96          SourceCode sourceCode = new SourceCode(
97                  new SourceCode.StringCodeLoader(
98                      "package foo.bar.baz;" +
99                      PMD.EOL +
100                     "/*****" +
101                     PMD.EOL +
102                     " * ugh" +
103                     PMD.EOL +
104                     " *****/" +
105                     PMD.EOL +
106                     "public class Foo {}"
107                 ));
108         Tokens tokens = new Tokens();
109         t.tokenize(sourceCode, tokens);
110         assertEquals(6, tokens.size());
111     }
112 
113     @Test
114     public void testDiscardOneLineAnnotationWithParams() throws Throwable {
115         JavaTokenizer t = new JavaTokenizer();
116         t.setIgnoreAnnotations(true);
117 
118         SourceCode sourceCode = new SourceCode(
119                 new SourceCode.StringCodeLoader(
120                     "package foo.bar.baz;" +
121                     PMD.EOL +
122                     "@ MyAnnotation (\"ugh\")" +
123                     PMD.EOL +
124                     "@NamedQueries({" +
125                             PMD.EOL +
126                     "@NamedQuery(" +
127                             PMD.EOL +
128                     ")})" +
129                             PMD.EOL +
130                     "public class Foo {" +
131                             PMD.EOL +
132                             "}"
133                 ));
134         Tokens tokens = new Tokens();
135         t.tokenize(sourceCode, tokens);
136         TokenEntry.getEOF();
137         System.out.println(sourceCode.getSlice(0,5));
138 
139         assertEquals(6, tokens.size());
140     }
141 
142     @Test
143     public void testIgnoreBetweenSpecialAnnotation() throws Throwable {
144         JavaTokenizer t = new JavaTokenizer();
145         t.setIgnoreAnnotations(false);
146         SourceCode sourceCode = new SourceCode(
147                 new SourceCode.StringCodeLoader(
148                     "package foo.bar.baz;" +
149                     PMD.EOL +
150                     "@SuppressWarnings({\"woof\",\"CPD-START\"})" +
151                     PMD.EOL +
152                     "@SuppressWarnings(\"CPD-START\")" +
153                     PMD.EOL +
154 
155                     "@ MyAnnotation (\"ugh\")" +
156                     PMD.EOL +
157                     "@NamedQueries({" +
158                             PMD.EOL +
159                     "@NamedQuery(" +
160                             PMD.EOL +
161                     ")})" +
162                             PMD.EOL +
163                     "public class Foo {}" +
164                     "@SuppressWarnings({\"ugh\",\"CPD-END\"})" +
165                     PMD.EOL
166 
167                 ));
168         Tokens tokens = new Tokens();
169         t.tokenize(sourceCode, tokens);
170         TokenEntry.getEOF();
171         assertEquals(10, tokens.size());
172 
173     }
174 
175 
176     public static junit.framework.Test suite() {
177         return new junit.framework.JUnit4TestAdapter(JavaTokensTokenizerTest.class);
178     }
179 }
180 
181