View Javadoc
1   /*******************************************************************************
2    * Copyright 2012 Internet2
3    * 
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * 
8    *   http://www.apache.org/licenses/LICENSE-2.0
9    * 
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   ******************************************************************************/
16  package edu.internet2.middleware.grouperInstaller;
17  
18  import java.io.BufferedInputStream;
19  import java.io.BufferedReader;
20  import java.io.ByteArrayInputStream;
21  import java.io.File;
22  import java.io.FileInputStream;
23  import java.io.FileOutputStream;
24  import java.io.FileReader;
25  import java.io.FilenameFilter;
26  import java.io.IOException;
27  import java.io.InputStream;
28  import java.io.InputStreamReader;
29  import java.io.OutputStream;
30  import java.net.MalformedURLException;
31  import java.net.URL;
32  import java.nio.file.Files;
33  import java.nio.file.Paths;
34  import java.nio.file.StandardOpenOption;
35  import java.security.SecureRandom;
36  import java.text.ParseException;
37  import java.text.SimpleDateFormat;
38  import java.util.ArrayList;
39  import java.util.Arrays;
40  import java.util.Collection;
41  import java.util.Collections;
42  import java.util.Date;
43  import java.util.Enumeration;
44  import java.util.HashMap;
45  import java.util.HashSet;
46  import java.util.Iterator;
47  import java.util.LinkedHashMap;
48  import java.util.LinkedHashSet;
49  import java.util.List;
50  import java.util.Map;
51  import java.util.Properties;
52  import java.util.Set;
53  import java.util.TreeMap;
54  import java.util.TreeSet;
55  import java.util.regex.Matcher;
56  import java.util.regex.Pattern;
57  import java.util.zip.ZipEntry;
58  import java.util.zip.ZipFile;
59  
60  import javax.xml.parsers.DocumentBuilder;
61  import javax.xml.parsers.DocumentBuilderFactory;
62  import javax.xml.xpath.XPath;
63  import javax.xml.xpath.XPathConstants;
64  import javax.xml.xpath.XPathExpression;
65  import javax.xml.xpath.XPathFactory;
66  
67  import org.w3c.dom.Document;
68  import org.w3c.dom.Element;
69  import org.w3c.dom.NamedNodeMap;
70  import org.w3c.dom.Node;
71  import org.w3c.dom.NodeList;
72  
73  import edu.internet2.middleware.grouperInstaller.GrouperInstaller.GrouperDirectories.GrouperInstallType;
74  import edu.internet2.middleware.grouperInstaller.GrouperInstallerIndexFile.PatchFileType;
75  import edu.internet2.middleware.grouperInstaller.morphString.Crypto;
76  import edu.internet2.middleware.grouperInstaller.util.GiDbUtils;
77  import edu.internet2.middleware.grouperInstaller.util.GrouperInstallerUtils;
78  import edu.internet2.middleware.grouperInstaller.util.GrouperInstallerUtils.CommandResult;
79  import edu.internet2.middleware.grouperInstallerExt.org.apache.commons.compress.archivers.tar.TarArchiveEntry;
80  import edu.internet2.middleware.grouperInstallerExt.org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
81  import edu.internet2.middleware.grouperInstallerExt.org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
82  import edu.internet2.middleware.grouperInstallerExt.org.apache.commons.compress.utils.IOUtils;
83  import edu.internet2.middleware.grouperInstallerExt.org.apache.commons.httpclient.HttpClient;
84  import edu.internet2.middleware.grouperInstallerExt.org.apache.commons.httpclient.methods.GetMethod;
85  
86  /**
87   * Install grouper
88   * @author mchyzer
89   *
90   */
91  public class GrouperInstaller {
92  
93    public static final String JAVA_MIN_VERSION = "1.7";
94  
95    /**
96     * location of where grouper is an find all the files within
97     */
98    private GrouperDirectories grouperDirectories = new GrouperDirectories();
99    
100   /**
101    * structure and logic to locate where grouper is installed (or will be?)
102    */
103   public static class GrouperDirectories {
104 
105     /**
106      * which type of install are we
107      */
108     public static enum GrouperInstallType {
109 
110       /**
111        * installed Grouper env
112        */
113       installed,
114       
115       /**
116        * source Grouper env
117        */
118       source;
119       
120     }
121 
122     /**
123      * are we an installed directory or source directory
124      */
125     private GrouperInstallType grouperInstallType;
126 
127     
128     /**
129      * are we an installed directory or source directory
130      * @return the grouperInstallType
131      */
132     public GrouperInstallType getGrouperInstallType() {
133       return this.grouperInstallType;
134     }
135 
136     
137     /**
138      * are we an installed directory or source directory
139      * @param grouperInstallType1 the grouperInstallType to set
140      */
141     public void setGrouperInstallType(GrouperInstallType grouperInstallType1) {
142       this.grouperInstallType = grouperInstallType1;
143     }
144 
145     
146     
147   }
148   
149   /**
150    * default ip address to listen for stuff
151    */
152   private String defaultIpAddress = null;
153   
154 
155   /**
156    * if we should continue or not
157    * @return if should continue
158    * @param autorunPropertiesKey key in properties file to automatically fill in a value
159    */
160   private static boolean shouldContinue(String autorunPropertiesKey) {
161     return shouldContinue(null, null, autorunPropertiesKey);
162   }
163   
164   /**
165    * if we should continue or not
166    * @param hint 
167    * @param exitHint 
168    * @param autorunPropertiesKey key in properties file to automatically fill in a value
169    * @return if should continue
170    */
171   private static boolean shouldContinue(String hint, String exitHint, String autorunPropertiesKey) {
172     if (hint == null) {
173       hint = "Do you want to continue ";
174     }
175     if (!hint.endsWith(" ")) {
176       hint += " ";
177     }
178     System.out.print(hint + "(t|f)? [f] ");
179     boolean shouldContinue = readFromStdInBoolean(false, autorunPropertiesKey);
180     if (!shouldContinue) {
181       if (exitHint == null) {
182         exitHint = "OK, will not continue, exiting...";
183       }
184       if (!GrouperInstallerUtils.isBlank(exitHint)) {
185         System.out.println(exitHint);
186       }
187     }
188     return shouldContinue;
189   }
190   
191   /**
192    * read a string from stdin
193    * @param defaultBoolean null for none, or true of false for if the input is blank
194    * @param autorunPropertiesKey key in properties file to automatically fill in a value
195    * @return the string
196    */
197   private static boolean readFromStdInBoolean(Boolean defaultBoolean, String autorunPropertiesKey) {
198     int i=100;
199     //keep trying until we get it
200     while(true) {
201       String input = readFromStdIn(autorunPropertiesKey);
202       if (GrouperInstallerUtils.isBlank(input) && defaultBoolean != null) {
203         return defaultBoolean;
204       }
205       try {
206         boolean inputBoolean = GrouperInstallerUtils.booleanValue(input);
207         return inputBoolean; 
208       } catch (Exception e) {
209         if (defaultBoolean != null) {
210           System.out.print("Expecting t or f or <blank> but received: '" + input + "', please try again: ");
211         } else {
212           System.out.print("Expecting t or f but received: '" + input + "', please try again: ");
213         }
214       }
215       if (i-- < 0) {
216         throw new RuntimeException("Too many tries for finding a boolean!");
217       }
218     }
219   }
220   
221   /**
222    * read a string from stdin
223    * @param autorunPropertiesKey key in properties file to automatically fill in a value
224    * @return the string
225    */
226   private static String readFromStdIn(String autorunPropertiesKey) {
227     
228     String str = null;
229     
230     if (GrouperInstallerUtils.propertiesContainsKey(autorunPropertiesKey)) {
231 
232       str = GrouperInstallerUtils.propertiesValue(autorunPropertiesKey, false);
233       
234       System.out.println("<using autorun property " + autorunPropertiesKey + ">: '" + str + "'");
235       
236     } else if (GrouperInstallerUtils.propertiesValueBoolean("grouperInstaller.autorun.useDefaultsAsMuchAsAvailable", false, false)) {
237       
238       System.out.println("<using default which is blank due to grouperInstaller.autorun.useDefaultsAsMuchAsAvailable and " + autorunPropertiesKey + ">: ");
239       
240     } else {
241 
242       if (GrouperInstallerUtils.propertiesValueBoolean("grouperInstaller.print.autorunKeys", false, false)) {
243         System.out.print("<" + autorunPropertiesKey + ">: ");
244       }
245       
246       try {
247         BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
248         str = in.readLine();
249       } catch (Exception e) {
250         throw new RuntimeException("Problem", e);
251       }
252 
253     }
254     
255     return GrouperInstallerUtils.trim(str);
256     
257   }
258 
259   /**
260    * download a file, delete the local file if it exists
261    * @param url
262    * @param localFileName
263    * @param autorunUseLocalFilePropertiesKey key in properties file to automatically fill in a value if download exists, default true
264    */
265   private void downloadFile(String url, String localFileName, String autorunUseLocalFilePropertiesKey) {
266     downloadFile(url, localFileName, false, null, autorunUseLocalFilePropertiesKey);
267   }
268 
269   /**
270    * download a file, delete the local file if it exists
271    * @param url
272    * @param localFileName
273    * @param allow404
274    * @param prefixFor404 print message prefix if 404
275    * @param autorunUseLocalFilePropertiesKey key in properties file to automatically fill in a value if download exists, default true
276    * @return true if downloaded, false if not
277    */
278   private boolean downloadFile(final String url, final String localFileName, final boolean allow404, 
279       final String prefixFor404, final String autorunUseLocalFilePropertiesKey) {
280 
281     boolean useLocalFile = false;
282 
283     final File localFile = new File(localFileName);
284 
285     if (localFile.exists()) {
286       
287       if (this.useAllLocalFiles != null && this.useAllLocalFiles == true) {
288         useLocalFile = true;
289       } else {
290         System.out.print("File exists: " + localFile.getAbsolutePath() + ", should we use the local file (t|f)? [t]: ");
291         useLocalFile = readFromStdInBoolean(true, autorunUseLocalFilePropertiesKey);
292         
293         if (useLocalFile && this.useAllLocalFiles == null) {
294           System.out.print("Would you like to use all local files (t|f)? [t]: ");
295           this.useAllLocalFiles = readFromStdInBoolean(true, "grouperInstaller.autorun.useAllLocalFiles");
296         }
297       }
298     }
299     
300     if (useLocalFile) {
301       return true;
302     }
303 
304     if (allow404 && GrouperInstallerUtils.propertiesValueBoolean("grouperInstaller.useLocalFilesOnlyForDevelopment", false, false)) {
305       return false;
306     }
307 
308     final boolean[] result = new boolean[1];
309     
310     Runnable runnable = new Runnable() {
311       
312       public void run() {
313         result[0] = downloadFileHelper(url, localFileName, allow404, prefixFor404, autorunUseLocalFilePropertiesKey);
314       }
315     };
316     
317     GrouperInstallerUtils.threadRunWithStatusDots(runnable, true);
318     
319     return result[0];
320   }
321   
322   /**
323    * download a file, delete the local file if it exists
324    * @param url
325    * @param localFileName
326    * @param allow404
327    * @param prefixFor404 print message prefix if 404
328    * @param autorunUseLocalFilePropertiesKey key in properties file to automatically fill in a value if download exists, default true
329    * @return true if downloaded, false if not
330    */
331   private static boolean downloadFileHelper(String url, String localFileName, boolean allow404, 
332       String prefixFor404, String autorunUseLocalFilePropertiesKey) {
333 
334     final File localFile = new File(localFileName);
335 
336     HttpClient httpClient = new HttpClient();
337 
338     String proxyServer = GrouperInstallerUtils.propertiesValue("download.server.proxyServer", false); 
339     
340     if (!GrouperInstallerUtils.isBlank(proxyServer)) {
341       
342       int proxyPort = GrouperInstallerUtils.propertiesValueInt("download.server.proxyPort", -1, true);
343       httpClient.getHostConfiguration().setProxy(proxyServer, proxyPort);
344     }
345     
346     //see if we are working with local files:
347     {
348       File localFileFromUrl = new File(url);
349       if (localFileFromUrl.exists()) {
350         System.out.println("Copying local file: " + url + " to file: " + localFileName);
351         
352         if (localFile.exists()) {
353           
354           System.out.println("File exists: " + localFile.getAbsolutePath() + ", deleting");
355           
356           if (!localFile.delete()) {
357             throw new RuntimeException("Cant delete file: " + localFile.getAbsolutePath() + "!!!!!");
358           }
359         } else {
360           if (!localFile.getParentFile().exists()) {
361             GrouperInstallerUtils.mkdirs(localFile.getParentFile());
362           }
363 
364         }
365         
366         try {
367           FileOutputStream fileOutputStream = new FileOutputStream(localFile);
368           FileInputStream fileInputStream = new FileInputStream(localFileFromUrl);
369   
370           GrouperInstallerUtils.copy(fileInputStream, fileOutputStream);
371   
372           return true;
373         } catch (Exception exception) {
374           String errorMessage = "Error copying file: " + url;
375           System.out.println(errorMessage);
376           throw new RuntimeException(errorMessage, exception);
377         }
378 
379       }
380       
381       //if it doesnt exist, see if the parent dir exists
382       if (localFileFromUrl.getParentFile().exists()) {
383 
384         //the dir is there but no file...   hmmmm
385         if (allow404) {
386           if (GrouperInstallerUtils.isBlank(prefixFor404)) {
387             prefixFor404 = "File not found: ";
388           }
389           System.out.println(prefixFor404 + url);
390           return false;
391         }
392 
393         //weve got a problem
394         
395       }
396     }
397     
398     GetMethod getMethod = null;
399     try {
400       
401       getMethod = new GetMethod(url);
402       
403       int result = httpClient.executeMethod(getMethod);
404       
405       if (allow404 && result == 404) {
406         if (GrouperInstallerUtils.isBlank(prefixFor404)) {
407           prefixFor404 = "File not found: ";
408         }
409         System.out.println(prefixFor404 + url);
410         return false;
411       }
412       
413       System.out.println("Downloading from URL: " + url + " to file: " + localFileName);
414 
415       if (result != 200) {
416         throw new RuntimeException("Expecting 200 but received: " + result);
417       }
418       
419       InputStream inputStream = getMethod.getResponseBodyAsStream();
420 
421       if (localFile.exists()) {
422         
423         System.out.println("File exists: " + localFile.getAbsolutePath() + ", deleting");
424         
425         if (!localFile.delete()) {
426           throw new RuntimeException("Cant delete file: " + localFile.getAbsolutePath() + "!!!!!");
427         }
428       }
429       
430       if (!localFile.getParentFile().exists()) {
431         GrouperInstallerUtils.mkdirs(localFile.getParentFile());
432       }
433       
434       FileOutputStream fileOutputStream = new FileOutputStream(localFile);
435 
436 
437       GrouperInstallerUtils.copy(inputStream, fileOutputStream);
438 
439       return true;
440 
441     } catch (Exception exception) {
442       String errorMessage = "Error connecting to URL: " + url;
443       System.out.println(errorMessage);
444       throw new RuntimeException(errorMessage, exception);
445     }
446   }
447 
448   /**
449    * 
450    * @param ehcacheBaseFile
451    */
452   public void convertEhcacheBaseToProperties(File ehcacheBaseFile) {
453     //File ehcacheBaseBakFile = this.bakFile(ehcacheBaseFile);
454     //GrouperInstallerUtils.copyFile(existingFile, bakFile, true);
455     //System.out.println("Backing up: " + existingFile.getAbsolutePath() + " to: " + bakFile.getAbsolutePath());
456     
457     NodeList nodeList = GrouperInstallerUtils.xpathEvaluate(ehcacheBaseFile, "/ehcache/cache");
458     
459     Set<String> usedKeys = new HashSet<String>();
460     
461     for (int i=0;i<nodeList.getLength();i++) {
462       
463       Element element = (Element)nodeList.item(i);
464 
465       //  <cache  name="edu.internet2.middleware.grouper.internal.dao.hib3.Hib3MemberDAO.FindBySubject"
466       //      maxElementsInMemory="5000"
467       //      eternal="false"
468       //      timeToIdleSeconds="5"
469       //      timeToLiveSeconds="10"
470       //      overflowToDisk="false"  
471       //      statistics="false"
472       //  />
473 
474       
475       String name = element.getAttribute("name");
476       Integer maxElementsInMemory = GrouperInstallerUtils.intObjectValue(element.getAttribute("maxElementsInMemory"), true);
477       Boolean eternal = GrouperInstallerUtils.booleanObjectValue(element.getAttribute("eternal"));
478       Integer timeToIdleSeconds = GrouperInstallerUtils.intObjectValue(element.getAttribute("timeToIdleSeconds"), true);
479       Integer timeToLiveSeconds = GrouperInstallerUtils.intObjectValue(element.getAttribute("timeToLiveSeconds"), true);
480       Boolean overflowToDisk = GrouperInstallerUtils.booleanObjectValue(element.getAttribute("overflowToDisk"));
481       Boolean statistics = GrouperInstallerUtils.booleanObjectValue(element.getAttribute("statistics"));
482 
483       //any attributes we dont expect?
484       NamedNodeMap configuredNamedNodeMap = element.getAttributes();
485       //see which attributes are new or changed
486       for (int j=0;j<configuredNamedNodeMap.getLength();j++) {
487         Node configuredAttribute = configuredNamedNodeMap.item(j);
488         if (!configuredAttribute.getNodeName().equals("name")
489             && !configuredAttribute.getNodeName().equals("maxElementsInMemory")
490             && !configuredAttribute.getNodeName().equals("eternal")
491             && !configuredAttribute.getNodeName().equals("timeToIdleSeconds")
492             && !configuredAttribute.getNodeName().equals("timeToLiveSeconds")
493             && !configuredAttribute.getNodeName().equals("overflowToDisk")
494             && !configuredAttribute.getNodeName().equals("statistics")) {
495           throw new RuntimeException("Cant process attribute: '" + configuredAttribute.getNodeName() + "'");
496         }
497       }
498       
499       String key = convertEhcacheNameToPropertiesKey(name, usedKeys);
500       
501       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.name = edu.internet2.middleware.grouper.internal.dao.hib3.Hib3GroupDAO
502       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.maxElementsInMemory = 500
503       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.eternal = false
504       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.timeToIdleSeconds = 1
505       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.timeToLiveSeconds = 1
506       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.overflowToDisk = false
507       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.statistics = false
508       
509       System.out.println("cache.name." + key + ".name = " + name);
510       if (maxElementsInMemory != null) {
511         System.out.println("cache.name." + key + ".maxElementsInMemory = " + maxElementsInMemory);
512       }
513       if (eternal != null) {
514         System.out.println("cache.name." + key + ".eternal = " + eternal);
515       }
516       if (timeToIdleSeconds != null) {
517         System.out.println("cache.name." + key + ".timeToIdleSeconds = " + timeToIdleSeconds);
518       }
519       if (timeToLiveSeconds != null) {
520         System.out.println("cache.name." + key + ".timeToLiveSeconds = " + timeToLiveSeconds);
521       }
522       if (overflowToDisk != null) {
523         System.out.println("cache.name." + key + ".overflowToDisk = " + overflowToDisk);
524       }
525       if (statistics != null) {
526         System.out.println("cache.name." + key + ".statistics = " + statistics);
527       }
528       System.out.println("");
529     }
530 
531   }
532 
533   /**
534    * convert a name like: edu.internet2.middleware.grouper.internal.dao.hib3.Hib3GroupDAO
535    * to: edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO
536    * @param ehcacheName 
537    * @param usedKeys
538    * @return the key
539    */
540   private static String convertEhcacheNameToPropertiesKey(String ehcacheName, Set<String> usedKeys) {
541     
542     StringBuilder result = new StringBuilder();
543 
544     //strip off this beginning to get the keys a little smaller
545     if (ehcacheName.startsWith("edu.internet2.middleware.grouper.")) {
546       ehcacheName = ehcacheName.substring("edu.internet2.middleware.grouper.".length());
547     }
548     
549     for (int i=0; i<ehcacheName.length(); i++) {
550       
551       char curChar = ehcacheName.charAt(i);
552       
553       if (Character.isLetter(curChar) || Character.isDigit(curChar)) {
554         result.append(curChar);
555       } else {
556         result.append("_");
557       }
558       
559     }
560 
561     String resultString = result.toString();
562     if (!usedKeys.contains(resultString)) {
563       return resultString;
564     }
565     
566     for (int i=2;i<100;i++) {
567       String newResult = resultString + "_" + i;
568       if (!usedKeys.contains(newResult)) {
569         return newResult;
570       }
571     }
572     
573     throw new RuntimeException("Cant find name for " + ehcacheName);
574   }
575   
576   /**
577    * @param args
578    */
579   public static void main(String[] args) {
580     
581     
582     GrouperInstallerrInstaller.html#GrouperInstaller">GrouperInstaller grouperInstaller = new GrouperInstaller();
583     
584 //    grouperInstaller.reportOnConflictingJars("/Users/mchyzer/git/grouper_v2_6/grouper/temp/jarTest");
585 
586 //    File tommeDir = new File("/Users/vsachdeva/git/i2-grouper-new/grouper/grouper-misc/grouper-installer/container/tomee");
587 //    File serverXmlFile = new File(tommeDir.getAbsolutePath()
588 //        + File.separator + "conf" + File.separator + "server.xml");
589 //    
590 //    //<Connector port="8009" protocol="AJP/1.3" redirectPort="8443" />
591 //    String newConnectorString = "<Connector port=\"8009\" protocol=\"AJP/1.3\" redirectPort=\"8443\" "
592 //        + "tomcatAuthentication=\"false\" URIEncoding=\"UTF-8\" scheme=\"https\" secure=\"true\" />";
593 //    
594 //    System.out.println("Editing tomee config file: " + serverXmlFile.getAbsolutePath());
595     
596 //    editFile(serverXmlFile, "port=\"8009\"", new String[]{"<Connector"}, 
597 //        null, newConnectorString, "connector modifications in server.xml");
598     
599   //<Connector port="8009" protocol="AJP/1.3" redirectPort="8443" />
600     //editFile(serverXmlFile, "port=\"([\\d]+)\"", new String[]{"<Connector", "protocol=\"AJP/1.3\""}, null, "", "tomcat JK port");
601     
602     
603     
604 //    String str1 = new Crypto("Urg4lyLGawLdwEvByGmf").encrypt("admin123");
605 //    System.out.println(str1);
606 //    String str = new Crypto("Urg4lyLGawLdwEvByGmf").decrypt("Y8iDZWF8RdowIiCgVKNg9Q==");
607 //    System.out.println(str);
608 
609      grouperInstaller.mainLogic();
610 
611 //    grouperInstaller.upgradeExistingApplicationDirectoryString = "D:\\temp\\temp\\grouperJarCopyDest\\";
612 //    grouperInstaller.grouperBaseBakDir = "D:\\temp\\temp\\grouperJarBak\\";
613     
614 //    grouperInstaller.upgradeJars(new File("D:\\temp\\temp\\grouperJarCopySource"), new File("D:\\temp\\temp\\grouperJarCopyDest"));
615     
616 //    grouperInstaller.version = "2.5.0";
617 //    
618 //    grouperInstaller.grouperTarballDirectoryString = "D:\\temp\\temp\\grouperInstaller\\";
619 //    
620 //    grouperInstaller.grouperInstallDirectoryString = "D:\\temp\\temp\\grouperInstaller\\install\\";
621     
622     //new GrouperInstaller().convertEhcacheBaseToProperties(new File("/Users/mchyzer/git/grouper_v2_3/grouper/conf/ehcache.example.xml"));
623 
624 //    GrouperInstaller.downloadFile("https://github.com/Internet2/grouper/archive/GROUPER_2_2_BRANCH.zip",
625 //        "C:\\app\\grouperInstallerTarballDir\\GROUPER_2_2_BRANCH.zip", false, null, 
626 //        "grouperInstaller.autorun.createPatchDownloadSourceUseLocalIfExist", true
627 //        );
628 
629 //    GrouperInstaller grouperInstaller = new GrouperInstaller();
630 //
631 //    grouperInstaller.upgradeExistingApplicationDirectoryString = "C:\\app\\grouper_2_1_installer\\grouper.ui-2.1.5\\dist\\grouper\\";
632 //  
633 //    grouperInstaller.grouperBaseBakDir = "C:\\app\\grouperInstallerTarballDir\\bak_UI_2014_10_26_15_06_19_957\\";
634 //      
635 //    grouperInstaller.copyFiles("C:\\app\\grouperInstallerTarballDir\\grouper.ui-2.2.1\\dist\\grouper\\",
636 //        "C:\\app\\grouper_2_1_installer\\grouper.ui-2.1.5\\dist\\grouper\\",
637 //        GrouperInstallerUtils.toSet("WEB-INF/lib", "WEB-INF/web.xml", "WEB-INF/classes"));
638 
639     
640    // copyFiles();
641     
642 //    
643 //    
644 //    grouperInstaller.dbUrl = 
645 //    grouperInstaller.dbUser = 
646 //    grouperInstaller.dbPass = 
647 //    grouperInstaller.giDbUtils = new GiDbUtils(grouperInstaller.dbUrl, grouperInstaller.dbUser, grouperInstaller.dbPass);
648 //    grouperInstaller.untarredApiDir = new File("c:/mchyzer/grouper/trunk/grouper-installer/grouper.apiBinary-2.0.2");
649 //    grouperInstaller.grouperInstallDirectoryString = "C:/mchyzer/grouper/trunk/grouper-installer/";
650 //    
651 //    grouperInstaller.version = "2.0.2";
652 //    grouperInstaller.grouperSystemPassword = "myNewPass";
653 //    
654 //    grouperInstaller.addDriverJarToClasspath();
655 ////      
656 //    grouperInstaller.checkDatabaseConnection();
657 ////      grouperInstaller.initDb();
658 ////      
659 ////      grouperInstaller.addQuickstartSubjects();
660 ////      grouperInstaller.addQuickstartData();
661 //    
662 ////    File uiDir = grouperInstaller.downloadUi();
663 ////    File unzippedUiFile = unzip(uiDir.getAbsolutePath());
664 ////    grouperInstaller.untarredUiDir = untar(unzippedUiFile.getAbsolutePath());
665 //
666 ////      grouperInstaller.configureUi();
667 ////
668 //      File antDir = grouperInstaller.downloadAnt();
669 //      File unzippedAntFile = unzip(antDir.getAbsolutePath());
670 //      grouperInstaller.untarredAntDir = untar(unzippedAntFile.getAbsolutePath());
671 ////
672 ////      grouperInstaller.buildUi();
673 //
674 //    File tomcatDir = grouperInstaller.downloadTomcat();
675 //    File unzippedTomcatFile = unzip(tomcatDir.getAbsolutePath());
676 //    grouperInstaller.untarredTomcatDir = untar(unzippedTomcatFile.getAbsolutePath());
677 //
678 //    grouperInstaller.configureTomcat();
679 //    
680 ////    grouperInstaller.configureTomcatUiWebapp();
681 ////    
682 //    grouperInstaller.tomcatConfigureGrouperSystem();
683 //    
684 ////    grouperInstaller.tomcatBounce("restart");
685 //
686 //    File wsDir = grouperInstaller.downloadWs();
687 //
688 //    File unzippedWsFile = unzip(wsDir.getAbsolutePath());
689 //    grouperInstaller.untarredWsDir = untar(unzippedWsFile.getAbsolutePath());
690 //    grouperInstaller.configureWs();
691 //    grouperInstaller.buildWs();
692 //    
693 //    grouperInstaller.configureTomcatWsWebapp();
694 //    grouperInstaller.tomcatBounce("restart");
695 //
696 //    File clientDir = grouperInstaller.downloadClient();
697 //    
698 //    File unzippedClientFile = unzip(clientDir.getAbsolutePath());
699 //    grouperInstaller.untarredClientDir = untar(unzippedClientFile.getAbsolutePath());
700 //    grouperInstaller.configureClient();
701 //
702 //    grouperInstaller.addGrouperSystemWsGroup();
703 //    
704 //    grouperInstaller.runClientCommand();
705 //    
706 //    //CommandResult commandResult = GrouperInstallerUtils.execCommand("cmd /c cd");
707 //    //String result = commandResult.getOutputText();
708 //    //
709 //    //System.out.println(result);
710 //
711 //
712 //    //editPropertiesFile(new File("C:\\mchyzer\\grouper\\trunk\\grouper-installer\\grouper.apiBinary-2.1.0\\conf\\grouper.hibernate.properties"), 
713 //    //    "hibernate.connection.password", "sdf");
714       
715     System.exit(0);
716   }
717 
718   /** e.g. 2.1.0 */
719   private String version;
720   
721   /**
722    * @param isInstallNotUpgrade install will bounce tomcat, configure, etc
723    * 
724    */
725   private void buildUi(boolean isInstallNotUpgrade) {
726     
727     File grouperUiBuildToDir = new File(this.grouperUiBuildToDirName());
728     
729     boolean rebuildUi = true;
730     
731     if (grouperUiBuildToDir.exists()) {
732       boolean defaultRebuild = GrouperInstallerUtils.propertiesValueBoolean("grouperInstaller.default.ui.rebuildIfBuilt", true, false);
733       System.out.print("The Grouper UI has been built in the past, do you want it rebuilt? (t|f) [" 
734           + (defaultRebuild ? "t" : "f") + "]: ");
735       rebuildUi = readFromStdInBoolean(defaultRebuild, "grouperInstaller.autorun.rebuildUiAfterHavingBeenBuilt");
736     }
737     
738     if (!rebuildUi) {
739       return;
740     }
741 
742     if (isInstallNotUpgrade) {
743       //stop tomcat
744       try {
745         tomcatBounce("stop");
746       } catch (Exception e) {
747         System.out.println("Couldnt stop tomcat, ignoring...");
748       }
749     }
750     
751     List<String> commands = new ArrayList<String>();
752     
753     addAntCommands(commands);
754     commands.add("dist");
755     
756     System.out.println("\n##################################");
757     System.out.println("Building UI with command:\n" + this.untarredUiDir.getAbsolutePath() + "> " 
758         + convertCommandsIntoCommand(commands) + "\n");
759     
760     CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
761         true, true, null, this.untarredUiDir, null, true);
762     
763     if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
764       System.out.println("stderr: " + commandResult.getErrorText());
765     }
766     if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
767       System.out.println("stdout: " + commandResult.getOutputText());
768     }
769     
770     if (isInstallNotUpgrade) {
771       System.out.print("Do you want to set the log dir of UI (t|f)? [t]: ");
772       boolean setLogDir = readFromStdInBoolean(true, "grouperInstaller.autorun.setLogDirOfUi");
773       
774       if (setLogDir) {
775         
776         ////set the log dir
777         //C:\apps\grouperInstallerTest\grouper.ws-2.0.2\grouper-ws\build\dist\grouper-ws\WEB-INF\classes\log4j.properties
778         //
779         //${grouper.home}logs
780   
781         String defaultLogDir = this.untarredTomcatDir.getAbsolutePath() + File.separator + "logs" + File.separator + "grouperUi";
782         System.out.print("Enter the UI log dir: [" + defaultLogDir + "]: ");
783         String logDir = readFromStdIn("grouperInstaller.autorun.uiLogDir");
784         logDir = GrouperInstallerUtils.defaultIfBlank(logDir, defaultLogDir);
785         
786         //lets replace \\ with /
787         logDir = GrouperInstallerUtils.replace(logDir, "\\\\", "/");
788         //lets replace \ with /
789         logDir = GrouperInstallerUtils.replace(logDir, "\\", "/");
790   
791         File log4jFile = new File(grouperUiBuildToDirName() + File.separator + "WEB-INF" + File.separator + "classes"
792             + File.separator + "log4j.properties");
793   
794         System.out.println("Editing file: " + log4jFile.getAbsolutePath());
795   
796         //log4j.appender.grouper_event.File = c:/apps/grouperInstallerTest/grouper.apiBinary-2.0.2/logs/grouper_event.log
797         editFile(log4jFile, "log4j\\.\\S+\\.File\\s*=\\s*([^\\s]+logs)/grouper_[^\\s]+\\.log", null, 
798             null, logDir, "UI log directory");
799   
800         File logDirFile = new File(defaultLogDir);
801         if (!logDirFile.exists()) {
802           System.out.println("Creating log directory: " + logDirFile.getAbsolutePath());
803           GrouperInstallerUtils.mkdirs(logDirFile);
804         }
805         //test log dir
806         File testLogDirFile = new File(logDirFile.getAbsolutePath() + File.separator + "testFile" + GrouperInstallerUtils.uniqueId() + ".txt");
807         GrouperInstallerUtils.saveStringIntoFile(testLogDirFile, "test");
808         if (!testLogDirFile.delete()) {
809           throw new RuntimeException("Cant delete file: " +  testLogDirFile.getAbsolutePath());
810         }
811         System.out.println("Created and deleted a test file successfully in dir: " + logDirFile.getAbsolutePath());
812       }
813     }    
814 
815     
816     System.out.println("\nEnd building UI");
817     System.out.println("##################################\n");
818 
819     
820   }
821 
822   /** ps command */
823   private String psCommandUnix;
824   
825   /**
826    * 
827    * @return the ps command in unix
828    */
829   private String psCommand() {
830     if (GrouperInstallerUtils.isWindows()) {
831       throw new RuntimeException("This is windows, why are you looking for sh???");
832     }
833     if (GrouperInstallerUtils.isBlank(this.psCommandUnix)) {
834       if (new File("/bin/ps").exists()) {
835         this.psCommandUnix = "/bin/ps";
836       } else if (new File("/usr/bin/ps").exists()) {
837         this.psCommandUnix = "/usr/bin/ps";
838       } else if (new File("/usr/local/bin/ps").exists()) {
839         this.psCommandUnix = "/usr/local/bin/ps";
840       } else {
841         throw new RuntimeException("Cant find 'ps' command!");
842       }
843     }
844     return this.psCommandUnix;
845   }
846 
847   /** grep command */
848   private String grepCommand;
849   
850   /**
851    * 
852    * @return the grep command in unix
853    */
854   private String grepCommand() {
855     if (GrouperInstallerUtils.isWindows()) {
856       throw new RuntimeException("This is windows, why are you looking for sh???");
857     }
858     if (GrouperInstallerUtils.isBlank(this.grepCommand)) {
859       if (new File("/bin/grep").exists()) {
860         this.grepCommand = "/bin/grep";
861       } else if (new File("/usr/bin/grep").exists()) {
862         this.grepCommand = "/usr/bin/grep";
863       } else if (new File("/usr/local/bin/grep").exists()) {
864         this.grepCommand = "/usr/local/bin/grep";
865       } else {
866         throw new RuntimeException("Cant find 'grep' command!");
867       }
868     }
869     return this.grepCommand;
870   }
871 
872   /** kill command */
873   private String killCommand;
874   
875   /**
876    * 
877    * @return the kill command in unix
878    */
879   private String killCommand() {
880     if (GrouperInstallerUtils.isWindows()) {
881       throw new RuntimeException("This is windows, why are you looking for sh???");
882     }
883     if (GrouperInstallerUtils.isBlank(this.killCommand)) {
884       if (new File("/bin/kill").exists()) {
885         this.killCommand = "/bin/kill";
886       } else if (new File("/usr/bin/kill").exists()) {
887         this.killCommand = "/usr/bin/kill";
888       } else if (new File("/usr/local/bin/kill").exists()) {
889         this.killCommand = "/usr/local/bin/kill";
890       } else {
891         throw new RuntimeException("Cant find 'kill' command!");
892       }
893     }
894     return this.killCommand;
895   }
896 
897   /** sh command */
898   private String shCommand;
899   
900   /**
901    * 
902    * @return the sh command in unix
903    */
904   private String shCommand() {
905     if (GrouperInstallerUtils.isWindows()) {
906       throw new RuntimeException("This is windows, why are you looking for sh???");
907     }
908     
909     if (!GrouperInstallerUtils.isBlank(this.shCommand)) {
910       return this.shCommand;
911     }
912     
913     String[] attempts = new String[]{
914         "bash", "/bin/bash", 
915         "/sbin/bash", "/usr/local/bin/bash", 
916         "/usr/bin/bash", "/usr/sbin/bash", 
917         "/usr/local/sbin/bash", "sh", "/bin/sh", 
918         "/sbin/sh", "/usr/local/bin/sh", 
919         "/usr/bin/sh", "/usr/sbin/sh", 
920         "/usr/local/sbin/sh"}; 
921     
922     for (String attempt : attempts) {
923     
924       try {
925         CommandResult commandResult = GrouperInstallerUtils.execCommand(
926             attempt, 
927             new String[]{"-version"}, true);
928         String shResult = commandResult.getOutputText();
929         if (GrouperInstallerUtils.isBlank(shResult)) {
930           shResult = commandResult.getErrorText();
931         }
932   
933         //if we get a result, thats good
934         if (!GrouperInstallerUtils.isBlank(shResult)) {
935           this.shCommand = attempt;
936           System.out.println("Using shell command: " + attempt);
937           return this.shCommand;
938         }
939         
940       } catch (Exception e) {
941         //this is ok, keep trying
942       }
943     }
944     //ok, we couldnt find it, 
945     System.out.print("Couldn't find the command 'sh'.  Enter the path of 'sh' (e.g. /bin/sh): ");
946     this.shCommand = readFromStdIn("grouperInstaller.autorun.pathOfShCommandIfNotFound");
947 
948     try {
949       CommandResult commandResult = GrouperInstallerUtils.execCommand(
950           this.shCommand, 
951           new String[]{"-version"}, true);
952       String shResult = commandResult.getOutputText();
953       if (GrouperInstallerUtils.isBlank(shResult)) {
954         shResult = commandResult.getErrorText();
955       }
956 
957       //if we get a result, thats good
958       if (!GrouperInstallerUtils.isBlank(shResult)) {
959         return this.shCommand;
960       }
961       
962     } catch (Exception e) {
963       throw new RuntimeException("Error: couldn't run: " + this.shCommand + " -version!", e);
964     }
965 
966     throw new RuntimeException("Error: couldn't run: " + this.shCommand + " -version!");
967     
968   }
969 
970   /**
971    * 
972    * @param commands
973    */
974   private void addGshCommands(List<String> commands) {
975     if (GrouperInstallerUtils.isWindows()) {
976       commands.add("cmd");
977       commands.add("/c");
978       commands.add(gshCommand());
979     } else {
980       //if you add this it messes up when args have spaces
981       //commands.add(shCommand());
982       commands.add(gshCommand());
983     }
984   }
985 
986   /**
987    * 
988    * @param commands
989    */
990   private void addAntCommands(List<String> commands) {
991     if (GrouperInstallerUtils.isWindows()) {
992       commands.add("cmd");
993       commands.add("/c");
994       commands.add(this.untarredAntDir.getAbsolutePath() + File.separator + "bin" + File.separator + "ant.bat");
995     } else {
996       commands.add(shCommand());
997       commands.add(this.untarredAntDir.getAbsolutePath() + File.separator + "bin" + File.separator + "ant");
998     }
999   }
1000 
1001   /**
1002    * 
1003    * @param commands
1004    */
1005   private void addMavenCommands(List<String> commands) {
1006     if (GrouperInstallerUtils.isWindows()) {
1007       commands.add("cmd");
1008       commands.add("/c");
1009       commands.add(this.untarredMavenDir.getAbsolutePath() + File.separator + "bin" + File.separator + "mvn.bat");
1010     } else {
1011       commands.add(shCommand());
1012       commands.add(this.untarredMavenDir.getAbsolutePath() + File.separator + "bin" + File.separator + "mvn");
1013     }
1014   }
1015   
1016   /**
1017    * @param arg
1018    * 
1019    */
1020   private void tomeeBounce(String arg) {
1021     
1022     if (!GrouperInstallerUtils.equals("start", arg) && !GrouperInstallerUtils.equals("stop", arg) && !GrouperInstallerUtils.equals("restart", arg)) {
1023       throw new RuntimeException("Expecting arg: start|stop|restart but received: " + arg);
1024     }
1025     
1026     if (GrouperInstallerUtils.equals("restart", arg)) {
1027       
1028       tomeeBounce("stop");
1029       tomeeBounce("start");
1030       return;
1031     }
1032     
1033     if (GrouperInstallerUtils.equals("stop", arg)) {
1034       
1035       if (GrouperInstallerUtils.portAvailable(this.tomeeHttpPort, this.defaultIpAddress)) {
1036         System.out.println("Tomee is supposed to be listening on port: " + this.tomeeHttpPort + ", port not listening, assuming tomee is not running...");
1037         if (!shouldContinue("Should we " + arg + " tomee anyway?", "", "grouperInstaller.autorun." + arg + "TomeeAnyway")) {
1038           return;
1039         }
1040       }
1041 
1042       
1043     } else {
1044       if (!GrouperInstallerUtils.portAvailable(this.tomeeHttpPort, this.defaultIpAddress)) {
1045         System.out.println("Tomee is supposed to be listening on port: " + this.tomeeHttpPort + ", port is already listening!!!!  Why is this????");
1046         if (!shouldContinue("Should we " + arg + " tomee anyway?", "", "grouperInstaller.autorun." + arg + "TomeeAnyway")) {
1047           return;
1048         }
1049       }
1050       
1051     }
1052     
1053     final List<String> commands = new ArrayList<String>();
1054     
1055     commands.add(getJavaCommand());
1056     commands.add("-Xmx640m");
1057     
1058     commands.add("-Dcatalina.home=" + this.untarredTomeeDir.getAbsolutePath());
1059     //commands.add("-Djava.util.logging.config.file=" + this.untarredTomcatDir.getAbsolutePath() + File.separator + "conf" + File.separator + "logging.properties");
1060     
1061     commands.add("-cp");
1062     commands.add(this.untarredTomeeDir.getAbsolutePath() + File.separator + "bin" + File.separator + "bootstrap.jar" + File.pathSeparator
1063         + this.untarredTomeeDir.getAbsolutePath() + File.separator + "bin" + File.separator + "tomcat-juli.jar");
1064     commands.add("org.apache.catalina.startup.Bootstrap");
1065     
1066     if (GrouperInstallerUtils.equals("stop", arg)) {
1067       commands.add("stop");
1068     }
1069     
1070     System.out.println("\n##################################");
1071     
1072     String command = "start".equals(arg) ? "startup" : "shutdown";
1073     
1074     System.out.println("Tomee " + arg + " with command (note you need CATALINA_HOME and JAVA_HOME set):\n  "
1075         + this.untarredTomeeDir.getAbsolutePath() + File.separator + "bin" + File.separator + command
1076         + (GrouperInstallerUtils.isWindows() ? ".bat" : ".sh") + "\n");
1077     
1078     //dont wait
1079     boolean waitFor = GrouperInstallerUtils.equals("stop", arg) ? true : false;
1080     
1081     if (waitFor) {
1082       try {
1083         CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
1084             true, true, null, 
1085             new File(this.untarredTomeeDir.getAbsolutePath() + File.separator + "bin"), null, true);
1086         
1087         if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
1088           System.out.println("stderr: " + commandResult.getErrorText());
1089         }
1090         if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
1091           System.out.println("stdout: " + commandResult.getOutputText());
1092         }
1093       } catch (Throwable e) {
1094         e.printStackTrace();
1095         if (!shouldContinue("grouperInstaller.autorun.continueAfterTomeeError")) {
1096           return;
1097         }
1098       }
1099     } else {
1100       //start in new thread
1101       Thread thread = new Thread(new Runnable() {
1102         
1103         @Override
1104         public void run() {
1105           GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
1106               true, true, null, 
1107               new File(GrouperInstaller.this.untarredTomeeDir.getAbsolutePath() + File.separator + "bin"), 
1108               GrouperInstaller.this.untarredTomeeDir.getAbsolutePath() + File.separator + "logs" + File.separator + "catalina", false);
1109         }
1110       });
1111       thread.setDaemon(true);
1112       thread.start();
1113 
1114     }
1115     
1116     System.out.println("\nEnd tomee " + arg + " (note: logs are in " + this.untarredTomeeDir.getAbsolutePath() + File.separator + "logs)");
1117     System.out.println("##################################\n");
1118 
1119     System.out.print("Should we check ports to see if tomee was able to " + arg + " (t|f)? [t]: ");
1120     
1121     boolean shouldCheckTomee = readFromStdInBoolean(true, "grouperInstaller.autorun." + arg + "TomeeCheckPorts");
1122     
1123     if (shouldCheckTomee) {
1124       System.out.print("Waiting for tomee to " + arg +  "...");
1125       boolean success = false;
1126       for (int i=0;i<60;i++) {
1127         GrouperInstallerUtils.sleep(1000);
1128         //check port
1129         boolean portAvailable = GrouperInstallerUtils.portAvailable(this.tomeeHttpPort, this.defaultIpAddress);
1130         if (GrouperInstallerUtils.equals("start", arg)) {
1131           if (!portAvailable) {
1132             success = true;
1133             System.out.println("\nTomee listening on port: " + this.tomeeHttpPort);
1134             break;
1135           }
1136         } else {
1137           if (portAvailable) {
1138             success = true;
1139             System.out.println("\nTomee not listening on port: " + this.tomeeHttpPort);
1140             break;
1141           }
1142         }
1143         System.out.print(".");
1144       }
1145       if (!success) {
1146         System.out.println("Trying to " + arg + " tomee but couldnt properly detect " + arg + " on port " + this.tomeeHttpPort);
1147         System.out.print("Press <enter> to continue... ");
1148         readFromStdIn("grouperInstaller.autorun.tomeePortProblem");
1149       }
1150     } else {
1151       System.out.println("Waiting 10 seconds for tomee to " + arg + "...");
1152       GrouperInstallerUtils.sleep(10000);
1153     }
1154   }
1155 
1156   
1157   /**
1158    * @param arg
1159    * 
1160    */
1161   private void tomcatBounce(String arg) {
1162     
1163     if (!GrouperInstallerUtils.equals("start", arg) && !GrouperInstallerUtils.equals("stop", arg) && !GrouperInstallerUtils.equals("restart", arg)) {
1164       throw new RuntimeException("Expecting arg: start|stop|restart but received: " + arg);
1165     }
1166     
1167     if (GrouperInstallerUtils.equals("restart", arg)) {
1168       
1169       tomcatBounce("stop");
1170       tomcatBounce("start");
1171       return;
1172     }
1173     
1174     if (GrouperInstallerUtils.equals("stop", arg)) {
1175       
1176       if (GrouperInstallerUtils.portAvailable(this.tomcatHttpPort, this.defaultIpAddress)) {
1177         System.out.println("Tomcat is supposed to be listening on port: " + this.tomcatHttpPort + ", port not listening, assuming tomcat is not running...");
1178         if (!shouldContinue("Should we " + arg + " tomcat anyway?", "", "grouperInstaller.autorun." + arg + "TomcatAnyway")) {
1179           return;
1180         }
1181       }
1182 
1183       
1184     } else {
1185       if (!GrouperInstallerUtils.portAvailable(this.tomcatHttpPort, this.defaultIpAddress)) {
1186         System.out.println("Tomcat is supposed to be listening on port: " + this.tomcatHttpPort + ", port is already listening!!!!  Why is this????");
1187         if (!shouldContinue("Should we " + arg + " tomcat anyway?", "", "grouperInstaller.autorun." + arg + "TomcatAnyway")) {
1188           return;
1189         }
1190       }
1191       
1192     }
1193     
1194     final List<String> commands = new ArrayList<String>();
1195     
1196 //    <java jar="${tomcat.home}/bin/bootstrap.jar" fork="true">
1197 //    03          <jvmarg value="-Dcatalina.home=${tomcat.home}"/>
1198 //    04      </java>
1199 //    05  </target>
1200 //    06   
1201 //    07  <target name="tomcat-stop">
1202 //    08      <java jar="${tomcat.home}/bin/bootstrap.jar" fork="true">
1203 //    09          <jvmarg value="-Dcatalina.home=${tomcat.home}"/>
1204 //    10          <arg line="stop"/>
1205 //    11      </java>
1206     
1207     commands.add(getJavaCommand());
1208     commands.add("-Xmx640m");
1209     
1210     commands.add("-Dcatalina.home=" + this.untarredTomcatDir.getAbsolutePath());
1211     //commands.add("-Djava.util.logging.config.file=" + this.untarredTomcatDir.getAbsolutePath() + File.separator + "conf" + File.separator + "logging.properties");
1212     
1213     //later versions of tomcat need the juli jar...
1214     if (new File(this.untarredTomcatDir.getAbsolutePath() + File.separator + "bin" + File.separator + "tomcat-juli.jar").exists()) {
1215       
1216       commands.add("-cp");
1217       commands.add(this.untarredTomcatDir.getAbsolutePath() + File.separator + "bin" + File.separator + "bootstrap.jar" + File.pathSeparator
1218           + this.untarredTomcatDir.getAbsolutePath() + File.separator + "bin" + File.separator + "tomcat-juli.jar");
1219       commands.add("org.apache.catalina.startup.Bootstrap");
1220     } else {
1221 
1222       commands.add("-jar");
1223       commands.add(this.untarredTomcatDir.getAbsolutePath() + File.separator + "bin" + File.separator + "bootstrap.jar");
1224     }
1225     
1226     if (GrouperInstallerUtils.equals("stop", arg)) {
1227       commands.add("stop");
1228     }
1229     
1230     System.out.println("\n##################################");
1231     
1232     String command = "start".equals(arg) ? "startup" : "shutdown";
1233     
1234     System.out.println("Tomcat " + arg + " with command (note you need CATALINA_HOME and JAVA_HOME set):\n  "
1235         + this.untarredTomcatDir.getAbsolutePath() + File.separator + "bin" + File.separator + command
1236         + (GrouperInstallerUtils.isWindows() ? ".bat" : ".sh") + "\n");
1237     
1238     //dont wait
1239     boolean waitFor = GrouperInstallerUtils.equals("stop", arg) ? true : false;
1240     
1241     if (waitFor) {
1242       try {
1243         CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
1244             true, true, null, 
1245             new File(this.untarredTomcatDir.getAbsolutePath() + File.separator + "bin"), null, true);
1246         
1247         if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
1248           System.out.println("stderr: " + commandResult.getErrorText());
1249         }
1250         if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
1251           System.out.println("stdout: " + commandResult.getOutputText());
1252         }
1253       } catch (Throwable e) {
1254         e.printStackTrace();
1255         if (!shouldContinue("grouperInstaller.autorun.continueAfterTomcatError")) {
1256           return;
1257         }
1258       }
1259     } else {
1260       //start in new thread
1261       Thread thread = new Thread(new Runnable() {
1262         
1263         @Override
1264         public void run() {
1265           GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
1266               true, true, null, 
1267               new File(GrouperInstaller.this.untarredTomcatDir.getAbsolutePath() + File.separator + "bin"), 
1268               GrouperInstaller.this.untarredTomcatDir.getAbsolutePath() + File.separator + "logs" + File.separator + "catalina", false);
1269         }
1270       });
1271       thread.setDaemon(true);
1272       thread.start();
1273 
1274     }
1275     
1276     System.out.println("\nEnd tomcat " + arg + " (note: logs are in " + this.untarredTomcatDir.getAbsolutePath() + File.separator + "logs)");
1277     System.out.println("##################################\n");
1278 
1279     System.out.print("Should we check ports to see if tomcat was able to " + arg + " (t|f)? [t]: ");
1280     
1281     boolean shouldCheckTomcat = readFromStdInBoolean(true, "grouperInstaller.autorun." + arg + "TomcatCheckPorts");
1282     
1283     if (shouldCheckTomcat) {
1284       System.out.print("Waiting for tomcat to " + arg +  "...");
1285       boolean success = false;
1286       for (int i=0;i<60;i++) {
1287         GrouperInstallerUtils.sleep(1000);
1288         //check port
1289         boolean portAvailable = GrouperInstallerUtils.portAvailable(this.tomcatHttpPort, this.defaultIpAddress);
1290         if (GrouperInstallerUtils.equals("start", arg)) {
1291           if (!portAvailable) {
1292             success = true;
1293             System.out.println("\nTomcat listening on port: " + this.tomcatHttpPort);
1294             break;
1295           }
1296         } else {
1297           if (portAvailable) {
1298             success = true;
1299             System.out.println("\nTomcat not listening on port: " + this.tomcatHttpPort);
1300             break;
1301           }
1302         }
1303         System.out.print(".");
1304       }
1305       if (!success) {
1306         System.out.println("Trying to " + arg + " tomcat but couldnt properly detect " + arg + " on port " + this.tomcatHttpPort);
1307         System.out.print("Press <enter> to continue... ");
1308         readFromStdIn("grouperInstaller.autorun.tomcatPortProblem");
1309       }
1310     } else {
1311       System.out.println("Waiting 10 seconds for tomcat to " + arg + "...");
1312       GrouperInstallerUtils.sleep(10000);
1313     }
1314   }
1315   
1316   /** db url */
1317   private String dbUrl;
1318 
1319   /** db user */
1320   private String dbUser;
1321 
1322   /** db pass */
1323   private String dbPass;
1324 
1325   /** untarred dir */
1326   private File untarredApiDir;
1327 
1328   /** untarred dir */
1329   private File untarredUiDir;
1330 
1331   /** untarred dir */
1332   private File untarredWsDir;
1333 
1334   /** untarred dir */
1335   private File untarredAntDir;
1336 
1337   /** untarred dir */
1338   private File untarredMavenDir;
1339 
1340   /** untarred dir */
1341   private File untarredTomcatDir;
1342   
1343   /** untarred tomee dir */
1344   private File untarredTomeeDir;
1345   
1346   /** main install dir, must end in file separator */
1347   private String grouperTarballDirectoryString;
1348   
1349   /** main install dir, must end in file separator */
1350   private String grouperInstallDirectoryString;
1351   
1352   /** base bak dir for backing up files that are upgraded, ends in File separator */
1353   private String grouperBaseBakDir;
1354   
1355   /** grouper system password */
1356   private String grouperSystemPassword;
1357   
1358   /**
1359    * 
1360    */
1361   private void tomeeConfigureGrouperSystem() {
1362     
1363 //    while (true) {
1364 //      System.out.print("Enter the GrouperSystem password: ");
1365 //      this.grouperSystemPassword = readFromStdIn("grouperInstaller.autorun.grouperSystemPassword");
1366 //      this.grouperSystemPassword = GrouperInstallerUtils.defaultString(this.grouperSystemPassword);
1367 //      
1368 //      if (!GrouperInstallerUtils.isBlank(this.grouperSystemPassword)) {
1369 //        break;
1370 //      }
1371 //      System.out.println("The GrouperSystem password cannot be blank!");
1372 //    }
1373 
1374     System.out.print("Do you want to set the GrouperSystem password in " + this.untarredTomeeDir + File.separator + "conf" + File.separator + "tomcat-users.xml? [t]: ");
1375     boolean setGrouperSystemPassword = readFromStdInBoolean(true, "grouperInstaller.autorun.setGrouperSystemPasswordInTomeeUsers");
1376     if (setGrouperSystemPassword) {
1377 
1378       //write to the tomee_users file
1379       //get the password
1380       File tomeeUsersXmlFile = new File(this.untarredTomeeDir.getAbsolutePath() + File.separator + "conf" + File.separator + "tomcat-users.xml");
1381       String existingPassword = GrouperInstallerUtils.xpathEvaluateAttribute(tomeeUsersXmlFile, "tomcat-users/user[@username='GrouperSystem']", "password");
1382       
1383       System.out.println("Editing file: " + tomeeUsersXmlFile.getAbsolutePath());
1384 
1385       NodeList existingRole = GrouperInstallerUtils.xpathEvaluate(tomeeUsersXmlFile, "tomcat-users/role");
1386       
1387       //<role rolename="grouper_user"/>
1388       //<user username="GrouperSystem" password="chang3m3" roles="grouper_user"/>
1389 
1390       
1391       if (existingPassword == null) {
1392 
1393         addToXmlFile(tomeeUsersXmlFile, ">",  new String[]{"<tomcat-users"}, "<user username=\"GrouperSystem\" password=\"" 
1394             + this.grouperSystemPassword + "\" roles=\"grouper_user\"/>", "Tomcat user GrouperSystem");
1395          
1396       } else {
1397         
1398         if (GrouperInstallerUtils.equals(existingPassword, this.grouperSystemPassword)) {
1399           System.out.println("  - password is already set to that value, leaving file unchanged...");
1400 
1401         } else {
1402           
1403           editFile(tomeeUsersXmlFile, "password=\"([^\"]*)\"", new String[]{"<user", "username=\"GrouperSystem\""}, 
1404               null, this.grouperSystemPassword, "Tomcat password for user GrouperSystem");
1405           
1406         }
1407         
1408       }
1409 
1410       if (existingRole == null || existingRole.getLength() == 0) {
1411         
1412         //add the role
1413         addToXmlFile(tomeeUsersXmlFile, ">",  new String[]{"<tomcat-users"}, "<role rolename=\"grouper_user\"/>", "Tomcat role grouper_user");
1414         
1415       }
1416     }
1417     
1418   }
1419   
1420   /**
1421    * 
1422    */
1423   private void tomcatConfigureGrouperSystem() {
1424     
1425     while (true) {
1426       System.out.print("Enter the GrouperSystem password: ");
1427       this.grouperSystemPassword = readFromStdIn("grouperInstaller.autorun.grouperSystemPassword");
1428       this.grouperSystemPassword = GrouperInstallerUtils.defaultString(this.grouperSystemPassword);
1429       
1430       if (!GrouperInstallerUtils.isBlank(this.grouperSystemPassword)) {
1431         break;
1432       }
1433       System.out.println("The GrouperSystem password cannot be blank!");
1434     }
1435 
1436     System.out.print("Do you want to set the GrouperSystem password in " + this.untarredTomcatDir + File.separator + "conf" + File.separator + "tomcat-users.xml? [t]: ");
1437     boolean setGrouperSystemPassword = readFromStdInBoolean(true, "grouperInstaller.autorun.setGrouperSystemPasswordInTomcatUsers");
1438     if (setGrouperSystemPassword) {
1439 
1440       //write to the tomcat_users file
1441       //get the password
1442       File tomcatUsersXmlFile = new File(this.untarredTomcatDir.getAbsolutePath() + File.separator + "conf" + File.separator + "tomcat-users.xml");
1443       String existingPassword = GrouperInstallerUtils.xpathEvaluateAttribute(tomcatUsersXmlFile, "tomcat-users/user[@username='GrouperSystem']", "password");
1444       
1445       System.out.println("Editing file: " + tomcatUsersXmlFile.getAbsolutePath());
1446 
1447       NodeList existingRole = GrouperInstallerUtils.xpathEvaluate(tomcatUsersXmlFile, "tomcat-users/role");
1448       
1449       //<role rolename="grouper_user"/>
1450       //<user username="GrouperSystem" password="chang3m3" roles="grouper_user"/>
1451 
1452       
1453       if (existingPassword == null) {
1454 
1455         addToXmlFile(tomcatUsersXmlFile, ">",  new String[]{"<tomcat-users"}, "<user username=\"GrouperSystem\" password=\"" 
1456             + this.grouperSystemPassword + "\" roles=\"grouper_user\"/>", "Tomcat user GrouperSystem");
1457          
1458       } else {
1459         
1460         if (GrouperInstallerUtils.equals(existingPassword, this.grouperSystemPassword)) {
1461           System.out.println("  - password is already set to that value, leaving file unchanged...");
1462 
1463         } else {
1464           
1465           editFile(tomcatUsersXmlFile, "password=\"([^\"]*)\"", new String[]{"<user", "username=\"GrouperSystem\""}, 
1466               null, this.grouperSystemPassword, "Tomcat password for user GrouperSystem");
1467           
1468         }
1469         
1470       }
1471 
1472       if (existingRole == null || existingRole.getLength() == 0) {
1473         
1474         //add the role
1475         addToXmlFile(tomcatUsersXmlFile, ">",  new String[]{"<tomcat-users"}, "<role rolename=\"grouper_user\"/>", "Tomcat role grouper_user");
1476         
1477       }
1478     }
1479     
1480   }
1481   
1482   /**
1483    * 
1484    */
1485   private void configureUi() {
1486     //build properties file
1487     File buildPropertiesFile = new File(this.untarredUiDir.getAbsolutePath() + File.separator + "build.properties");
1488     if (!buildPropertiesFile.exists()) {
1489       File buildPropertiesTemplateFile = new File(this.untarredUiDir.getAbsolutePath() + File.separator + "build.properties.template");
1490       System.out.println("Copying file: " + buildPropertiesTemplateFile.getAbsolutePath() + " to file: " + buildPropertiesFile);
1491       GrouperInstallerUtils.copyFile(buildPropertiesTemplateFile, buildPropertiesFile, true);
1492     }
1493     
1494     //set the grouper property
1495     System.out.println("Editing " + buildPropertiesFile.getAbsolutePath() + ": ");
1496     String apiDir = GrouperInstallerUtils.replace(this.untarredApiDir.getAbsolutePath(),"\\\\", "/");
1497     apiDir = GrouperInstallerUtils.replace(apiDir, "\\", "/");
1498     editPropertiesFile(buildPropertiesFile, "grouper.folder", apiDir, false);
1499     editPropertiesFile(buildPropertiesFile, "should.copy.context.xml.to.metainf", "false", false);
1500     
1501   }
1502   
1503   /**
1504    * 
1505    */
1506   private void configureWs() {
1507     //build properties file
1508     File buildPropertiesFile = new File(this.untarredWsDir.getAbsolutePath() + File.separator 
1509         + "grouper-ws" + File.separator + "build.properties");
1510     if (!buildPropertiesFile.exists()) {
1511       File buildPropertiesTemplateFile = new File(this.untarredWsDir.getAbsolutePath() 
1512           + File.separator + "grouper-ws" + File.separator + "build.example.properties");
1513       System.out.println("Copying file: " + buildPropertiesTemplateFile.getAbsolutePath() + " to file: " + buildPropertiesFile);
1514       GrouperInstallerUtils.copyFile(buildPropertiesTemplateFile, buildPropertiesFile);
1515     }
1516     
1517     //set the grouper property
1518     System.out.println("Editing " + buildPropertiesFile.getAbsolutePath() + ": ");
1519     String apiDir = GrouperInstallerUtils.replace(this.untarredApiDir.getAbsolutePath(),"\\\\", "/");
1520     apiDir = GrouperInstallerUtils.replace(apiDir, "\\", "/");
1521     editPropertiesFile(buildPropertiesFile, "grouper.dir", apiDir, false);
1522     
1523   }
1524 
1525   /**
1526    * main function of grouper installer
1527    */
1528   public static enum GrouperInstallerMainFunction {
1529     
1530     /** install grouper */
1531     admin {
1532 
1533       @Override
1534       public void logic(GrouperInstaller grouperInstaller) {
1535         
1536         grouperInstaller.mainAdminLogic();
1537 
1538       }
1539     },
1540 
1541     /** install grouper */
1542     @Deprecated //TODO delete it.
1543     install {
1544 
1545       @Override
1546       public void logic(GrouperInstaller grouperInstaller) {
1547         
1548         grouperInstaller.mainInstallLogic();
1549 
1550       }
1551     },
1552     
1553     /** upgrade grouper */
1554     upgrade {
1555 
1556       @Override
1557       public void logic(GrouperInstaller grouperInstaller) {
1558         
1559         grouperInstaller.mainUpgradeLogic();
1560 
1561       }
1562     },
1563     
1564     /** create patch */
1565     createPatch {
1566 
1567       @Override
1568       public void logic(GrouperInstaller grouperInstaller) {
1569         
1570         grouperInstaller.mainCreatePatchLogic();
1571 
1572       }
1573     },
1574     
1575     /** see if there are patches available for grouper */
1576     patch {
1577 
1578       @Override
1579       public void logic(GrouperInstaller grouperInstaller) {
1580         
1581         grouperInstaller.mainPatchLogic();
1582 
1583       }
1584     },
1585     
1586     /** build container */  
1587     buildContainer {
1588 
1589       @Override
1590       public void logic(GrouperInstaller grouperInstaller) {
1591         grouperInstaller.mainBuildContainerLogic();
1592         
1593       }
1594       
1595     },
1596     
1597     /** install container */  
1598     installContainer {
1599 
1600       @Override
1601       public void logic(GrouperInstaller grouperInstaller) {
1602         grouperInstaller.mainInstallContainerLogic();
1603         
1604       }
1605       
1606     }
1607     ;
1608 
1609     /**
1610      * run the logic for the installer function
1611      * @param grouperInstaller
1612      */
1613     public abstract void logic(GrouperInstaller grouperInstaller);
1614     
1615     /**
1616      * 
1617      * @param string
1618      * @param exceptionIfInvalid
1619      * @param exceptionIfBlank
1620      * @return the action
1621      */
1622     public static GrouperInstallerMainFunction valueOfIgnoreCase(String string, boolean exceptionIfBlank, boolean exceptionIfInvalid) {
1623       return GrouperInstallerUtils.enumValueOfIgnoreCase(GrouperInstallerMainFunction.class, string, exceptionIfBlank, exceptionIfInvalid);
1624     }
1625 
1626 
1627     /**
1628      * convert a string to the enum
1629      * @param theString
1630      * @param exceptionOnInvalid
1631      * @return the enum
1632      */
1633     public static GrouperInstallerMainFunction valueOfIgnoreCase(String theString, boolean exceptionOnInvalid) {
1634       return GrouperInstallerUtils.enumValueOfIgnoreCase(GrouperInstallerMainFunction.class, theString, false, exceptionOnInvalid);
1635     }
1636   }
1637 
1638   private static String javaCommand = null;
1639 
1640   /**
1641    * When the installer starts, it tests and validates the java command. If found it will set the path to it for later use.
1642    *
1643    * @return path to java executable. Set by {@link #validJava()}
1644    */
1645   public static String getJavaCommand() {
1646     if (javaCommand != null) {
1647       return javaCommand;
1648     } else {
1649       throw new RuntimeException("Unable to determine \"java\" command to execute");
1650     }
1651   }
1652 
1653   private static String getJavaCommand(boolean throwIfNull) {
1654     if (throwIfNull) {
1655       return getJavaCommand();
1656     } else {
1657       return javaCommand;
1658     }
1659   }
1660 
1661   private static void setJavaCommand(String theJavaCommand) {
1662     //System.out.println("INFO: external java command set to '" + theJavaCommand + "'");
1663     javaCommand = theJavaCommand;
1664   }
1665 
1666   /**
1667    * Validate java and javac commands through various methods - using JAVA_HOME/bin/(cmd) and shell path location
1668    */
1669   private static void validJava() {
1670     boolean hadError = false;
1671     boolean cmdHadError = false;
1672     String command = null;
1673 
1674     // the most important check is the JAVA_HOME is set properly; this will be needed for any ant or maven commands, etc.
1675     String javaHome = System.getenv("JAVA_HOME");
1676 
1677     if (GrouperInstallerUtils.isBlank(javaHome)) {
1678       System.out.println("Non-fatal ERROR: you should have the environment variable JAVA_HOME set to a " + JAVA_MIN_VERSION + "+ JDK (currently not set)");
1679       hadError = true;
1680     } else {
1681       // try JAVA_HOME/bin/java
1682       command = javaHome + File.separator + "bin" + File.separator + "java";
1683       cmdHadError = validJavaOutput(command, "$JAVA_HOME/bin/java", false, false);
1684       if (getJavaCommand(false) == null && !cmdHadError) {
1685         setJavaCommand(command);
1686       }
1687       hadError = hadError || cmdHadError;
1688 
1689       // try JAVA_HOME/bin/javac
1690       command = javaHome + File.separator + "bin" + File.separator + "javac";
1691       cmdHadError = validJavaOutput(command, "$JAVA_HOME/bin/javac", true, false);
1692       hadError = hadError || cmdHadError;
1693     }
1694 
1695     // try bare "java" with shell path resolver
1696     command = "java";
1697     cmdHadError = validJavaOutput(command, "java command in the PATH", false, false);
1698     if (getJavaCommand(false) == null && !cmdHadError) {
1699       setJavaCommand(command);
1700     }
1701     hadError = hadError || cmdHadError;
1702 
1703     // try bare "javac" with shell path resolver
1704     command = "javac";
1705     cmdHadError = validJavaOutput(command, "javac command in the PATH", true, false);
1706     hadError = hadError || cmdHadError;
1707 
1708     // if nothing else works, the effective home for the java command running the installer is a minimal catch-all, suitable
1709     // for running simple tasks like starting Tomcat
1710     if (getJavaCommand(false) == null) {
1711       command = System.getProperty("java.home") + File.separator + "bin" + File.separator + "java";
1712       cmdHadError = validJavaOutput(command, "the current java command running the installer", false, false);
1713       if (!cmdHadError) {
1714         setJavaCommand(command);
1715       }
1716       hadError = hadError || cmdHadError;
1717     }
1718 
1719     if (hadError) {
1720       // assume all the above checks output something on error, otherwise the line below won't make sense
1721       System.out.println("WARNING: JAVA_HOME or Java path errors may cause issues when running external commands - these should be fixed before continuing.");
1722       System.out.print("Press <enter> to continue... ");
1723       readFromStdIn("grouperInstaller.autorun.javaInvalid");
1724     }
1725   }
1726  
1727   /**
1728    * take a java command (e.g. java, or javac, or %JAVA_HOME%/bin/java and make sure version is valid
1729    * @param command
1730    * @param what
1731    * @param jdkTest
1732    * @param fatal
1733    * @return if error
1734    */
1735   private static boolean validJavaOutput(String command, String what, boolean jdkTest, boolean fatal) {
1736     
1737     boolean printStackOnError = GrouperInstallerUtils.propertiesValueBoolean("grouperInstaller.printStackOnJavaVersionErrors", false, false);
1738 
1739     try {
1740     
1741       List<String> commands = new ArrayList<String>();
1742       
1743       commands.add(command);
1744       commands.add("-version");
1745         
1746       CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
1747           true, true, null, null, null, false, true, printStackOnError);
1748       
1749       //note this is printed view stderr not stdout
1750       String output = commandResult.getErrorText();
1751 
1752       // find the first numeric match of a.b or a.b.c*
1753       Pattern javaVersionPattern = Pattern.compile(".*?[^\\d]*(\\d+\\.\\d+(\\.\\d+)?).*", Pattern.DOTALL);
1754       Matcher javaVersionMatcher = javaVersionPattern.matcher(output);
1755       if (!javaVersionMatcher.matches()) {
1756         output = commandResult.getOutputText();
1757         javaVersionMatcher = javaVersionPattern.matcher(output);
1758         
1759         if (!javaVersionMatcher.matches()) {
1760           if (jdkTest) {
1761             System.out.println((fatal ? "" : "Non-fatal ") + "ERROR: can't find 'javac' command in " + what + ", Java needs to be a JDK not a JRE!");
1762           }
1763           System.out.println((fatal ? "" : "Non-fatal ") + "ERROR trying to check java output, make sure you have " + what 
1764               + " set to Java " + (jdkTest ? "JDK (not JRE) " : "") + JAVA_MIN_VERSION + "+\n"
1765               + commandResult.getErrorText() + "\n" + commandResult.getOutputText());
1766           if (!fatal) {
1767             return true;
1768           }
1769           System.out.print("Press <enter> to continue... ");
1770           readFromStdIn("grouperInstaller.autorun.javaInvalid");
1771           System.exit(1);
1772         }
1773       }
1774       
1775       String versionString = javaVersionMatcher.group(1);
1776 
1777       if (GrouperInstallerUtils.compareVersions(versionString, JAVA_MIN_VERSION) < 0) {
1778         System.out.println((fatal ? "" : "Non-fatal ") + "ERROR: " + what
1779           + (jdkTest ? " requires to be" : " should be") + " invoked with Java " + JAVA_MIN_VERSION + "+"
1780           + (jdkTest ? " JDK" : "") + ", but was: " + versionString);
1781         if (!fatal) {
1782           return true;
1783         }
1784         System.out.print("Press <enter> to continue... ");
1785         readFromStdIn("grouperInstaller.autorun.javaInvalid");
1786         System.exit(1);
1787       }
1788       return false;
1789     } catch (RuntimeException re) {
1790 
1791       if (printStackOnError) {
1792         re.printStackTrace();
1793       }
1794 
1795       System.out.println((fatal ? "" : "Non-fatal ") + "ERROR trying to check java output, make sure you have " + what 
1796           + " set to Java " + (jdkTest ? "JDK (not JRE) " : "") + JAVA_MIN_VERSION + "+\n" + re.getMessage());
1797       return true;
1798     }
1799   }
1800 
1801   /**
1802    * 
1803    */
1804   private void mainLogic() {
1805     
1806     validJava();
1807     
1808     this.grouperInstallerMainFunction = this.grouperInstallerMainFunction();
1809     
1810     this.grouperInstallerMainFunction.logic(this);
1811     
1812   }
1813   
1814   /**
1815    * what are we doing
1816    */
1817   private GrouperInstallerMainFunction grouperInstallerMainFunction;
1818   
1819   /**
1820    * @param appDir e.g. this.upgradeExistingApplicationDirectoryString
1821    */
1822   public void reportOnConflictingJars(String appDir) {
1823     
1824     System.out.println("\n##################################");
1825     System.out.println("Looking for conflicting jars\n");
1826 
1827     //look for conflicting jars
1828     List<File> allLibraryJars = findAllLibraryFiles(appDir);
1829     
1830     System.out.println("Found " + GrouperInstallerUtils.length(allLibraryJars) + " jars");
1831     
1832     Set<String> alreadyProcessed = new HashSet<String>();
1833     
1834     for (File jarFile : new ArrayList<File>(allLibraryJars)) {
1835       try {
1836         if (!jarFile.exists()) {
1837           allLibraryJars.remove(jarFile);
1838           continue;
1839         }
1840         
1841          Set<String> baseNames = GrouperInstallerUtils.jarFileBaseNames(jarFile.getName());
1842         
1843         //dont print multiple times
1844         if (alreadyProcessed.containsAll(baseNames)) {
1845           continue;
1846         }
1847         
1848         alreadyProcessed.addAll(baseNames);
1849         
1850         List<File> relatedFiles = GrouperInstallerUtils.nonNull(GrouperInstallerUtils.jarFindJar(allLibraryJars, jarFile.getName()));
1851         Iterator<File> relatedFilesIterator = relatedFiles.iterator();
1852         
1853         while (relatedFilesIterator.hasNext()) {
1854           if (jarFile.equals(relatedFilesIterator.next())) {
1855             relatedFilesIterator.remove();
1856           }
1857         }
1858         
1859         if (GrouperInstallerUtils.length(relatedFiles) >= 1) {
1860           
1861           if (relatedFiles.size() == 1) {
1862             File relatedFile = relatedFiles.iterator().next();
1863             File newerVersion = GrouperInstallerUtils.jarNewerVersion(relatedFile, jarFile);
1864             if (newerVersion != null) {
1865               
1866               if (newerVersion.equals(jarFile)) {
1867                 System.out.println("There is a conflicting jar: " + jarFile.getAbsolutePath());
1868                 System.out.println("Deleting older jar: " + relatedFile.getAbsolutePath());
1869                 GrouperInstallerUtils.fileDelete(relatedFile);
1870                 allLibraryJars.remove(relatedFile);
1871               } else {
1872                 System.out.println("There is a conflicting jar: " + relatedFile.getAbsolutePath());
1873                 System.out.println("Deleting older jar: " + jarFile.getAbsolutePath());
1874                 GrouperInstallerUtils.fileDelete(jarFile);
1875                 allLibraryJars.remove(jarFile);
1876               }
1877               System.out.print("Press <enter> to continue... ");
1878               readFromStdIn("grouperInstaller.autorun.conflictingJarContinue");
1879               continue;
1880             }
1881           }
1882           
1883           System.out.println("There is a conflicting jar: " + GrouperInstallerUtils.toStringForLog(relatedFiles));
1884           System.out.println("You should probably delete one of these files (oldest one?) or consult the Grouper team.");
1885           System.out.print("Press <enter> to continue... ");
1886           readFromStdIn("grouperInstaller.autorun.conflictingJarContinue");
1887         }
1888         
1889   //      if (GrouperInstallerUtils.length(relatedFiles) == 0) {
1890   //        System.out.println("Why is jar file not found??? " + jarFile.getAbsolutePath());
1891   //      }
1892       } catch (RuntimeException re) {
1893         GrouperInstallerUtils.injectInException(re, "Problem with jar: " + jarFile.getAbsolutePath());
1894         throw re;
1895       }
1896     }
1897   }
1898   
1899   /**
1900    * which app is being upgraded
1901    */
1902   private AppToUpgrade appToUpgrade;
1903 
1904   /**
1905    * patch grouper
1906    */
1907   private void mainCreatePatchLogic() {
1908 
1909     //####################################
1910     //Find out what directory to upgrade to.  This ends in a file separator
1911     this.grouperTarballDirectoryString = grouperUpgradeTempDirectory();
1912 
1913     //see what we are upgrading: api, ui, ws, client
1914     this.appToUpgrade = grouperAppToUpgradeOrPatch("create a patch for");
1915     
1916     if (this.appToUpgrade == AppToUpgrade.CLIENT) {
1917       throw new RuntimeException("Cant create patches for client, just put the client patch files in an API patch");
1918     }
1919     
1920     String branchToCreatePatchFor = null;
1921     {
1922       String defaultBranchToCreatePatchFor = GrouperInstallerUtils.propertiesValue("grouperInstaller.default.branchToCreatePatchFor", false);
1923       
1924       if (GrouperInstallerUtils.isBlank(defaultBranchToCreatePatchFor)) {
1925         //grouper.version = 2.2.1
1926         // convert to GROUPER_2_2_BRANCH
1927         String grouperVersion = GrouperInstallerUtils.propertiesValue("grouper.version", true);
1928 
1929         grouperVersion = GrouperInstallerUtils.replace(grouperVersion, ".", "_");
1930 
1931         Pattern pattern = Pattern.compile("(\\d+_\\d+_)\\d+");
1932         Matcher matcher = pattern.matcher(grouperVersion);
1933         if (matcher.matches()) {
1934           String majorMinor = matcher.group(1);
1935           defaultBranchToCreatePatchFor = "GROUPER_" + majorMinor + "BRANCH";
1936         }
1937 
1938         
1939       }
1940       
1941       System.out.print("What branch do you want to create a patch for (e.g. GROUPER_2_2_BRANCH)? [" + defaultBranchToCreatePatchFor + "]: ");
1942       branchToCreatePatchFor = readFromStdIn("grouperInstaller.autorun.branchToCreatePatchFor");
1943       if (GrouperInstallerUtils.isBlank(branchToCreatePatchFor)) {
1944         branchToCreatePatchFor = defaultBranchToCreatePatchFor;
1945       }
1946     }
1947 
1948     String branchForPspToCreatePatchFor = null;
1949 
1950     if (this.appToUpgrade == AppToUpgrade.PSP) {
1951       String defaultBranchForPspToCreatePatchFor = GrouperInstallerUtils.propertiesValue("grouperInstaller.default.branchForPspToCreatePatchFor", false);
1952       
1953       if (GrouperInstallerUtils.isBlank(defaultBranchForPspToCreatePatchFor)) {
1954         //grouper.version = 2.2.1
1955         // convert to PSP_2_2_BRANCH
1956         String grouperVersion = GrouperInstallerUtils.propertiesValue("grouper.version", true);
1957 
1958         grouperVersion = GrouperInstallerUtils.replace(grouperVersion, ".", "_");
1959 
1960         Pattern pattern = Pattern.compile("(\\d+_\\d+_)\\d+");
1961         Matcher matcher = pattern.matcher(grouperVersion);
1962         if (matcher.matches()) {
1963           String majorMinor = matcher.group(1);
1964           defaultBranchForPspToCreatePatchFor = "PSP_" + majorMinor + "BRANCH";
1965         }
1966       }
1967       
1968       System.out.print("What PSP branch do you want to create a patch for (e.g. GROUPER_2_2_BRANCH)? [" + defaultBranchForPspToCreatePatchFor + "]: ");
1969       branchForPspToCreatePatchFor = readFromStdIn("grouperInstaller.autorun.branchForPspToCreatePatchFor");
1970       if (GrouperInstallerUtils.isBlank(branchForPspToCreatePatchFor)) {
1971         branchForPspToCreatePatchFor = defaultBranchForPspToCreatePatchFor;
1972       }
1973       
1974     }
1975     
1976     int nextPatchIndex = -1;
1977     
1978     {
1979       String grouperVersion = GrouperInstallerUtils.propertiesValue("grouper.version", true);
1980   
1981       grouperVersion = GrouperInstallerUtils.replace(grouperVersion, ".", "_");
1982   
1983       nextPatchIndex = this.downloadPatches(this.appToUpgrade, grouperVersion);
1984     }
1985     
1986     //see if dir is there: e.g. grouper_v2_2_1_ui_patch_0
1987     String patchName = null;
1988     
1989     {
1990       String grouperVersion = GrouperInstallerUtils.propertiesValue("grouper.version", true);
1991 
1992       grouperVersion = GrouperInstallerUtils.replace(grouperVersion, ".", "_");
1993 
1994       patchName = "grouper_v" + grouperVersion + "_" + this.appToUpgrade.name().toLowerCase() + "_patch_" + nextPatchIndex;
1995     }
1996  
1997     {
1998       System.out.println("Next patch index for " + this.appToUpgrade + " is " + nextPatchIndex + ". ok (" + patchName + ")? (t|f)? [t]:");
1999       boolean continueOn = readFromStdInBoolean(true, "grouperInstaller.autorun.patchIndexIsOk");
2000       if (!continueOn) {
2001         System.out.println("Patch index is not ok");
2002         throw new RuntimeException("Patch index is not ok");
2003       }
2004     }
2005     
2006     downloadAndUnzipGrouperSource(branchToCreatePatchFor);
2007 
2008     File sourceTagDir = null;
2009 
2010     {
2011       String grouperVersion = GrouperInstallerUtils.propertiesValue("grouper.version", true);
2012       String grouperTag = GrouperInstallerUtils.replace(grouperVersion, ".", "_");
2013       System.out.println("Using Grouper tag: " + grouperTag);
2014       downloadAndUnzipGrouperSource("GROUPER_" + grouperTag);
2015       
2016       sourceTagDir = new File(this.grouperTarballDirectoryString + "GROUPER_" + grouperTag
2017           + File.separator + "grouper-GROUPER_" + grouperTag);
2018       
2019     }
2020     
2021     //grouper is in downloadDir/GROUPER_2_2_BRANCH/grouper-GROUPER_2_2_BRANCH
2022     File sourceDir = new File(this.grouperTarballDirectoryString + branchToCreatePatchFor
2023         + File.separator + "grouper-" + branchToCreatePatchFor);
2024 
2025     if (!sourceDir.exists()) {
2026       throw new RuntimeException("Why does source dir not exist??? " + sourceDir);
2027     }
2028 
2029     //grouper is in downloadDir/GROUPER_2_2_BRANCH/grouper-GROUPER_2_2_BRANCH
2030     File pspSourceDir = null;
2031     File pspSourceTagDir = null;
2032 
2033     if (this.appToUpgrade == AppToUpgrade.PSP) {
2034       downloadAndUnzipPspSource(branchForPspToCreatePatchFor);
2035 
2036       pspSourceDir = new File(this.grouperTarballDirectoryString + branchForPspToCreatePatchFor
2037           + File.separator + "grouper-psp-" + branchForPspToCreatePatchFor);
2038 
2039       if (!pspSourceDir.exists()) {
2040         throw new RuntimeException("Why does PSP source dir not exist??? " + pspSourceDir);
2041       }
2042 
2043       String grouperVersion = GrouperInstallerUtils.propertiesValue("grouper.version", true);
2044       System.out.println("Using PSP tag: " + grouperVersion);
2045       downloadAndUnzipPspSource(grouperVersion);
2046 
2047       pspSourceTagDir = new File(this.grouperTarballDirectoryString + grouperVersion
2048           + File.separator + "grouper-psp-" + grouperVersion);
2049 
2050       if (!pspSourceTagDir.exists()) {
2051         throw new RuntimeException("Why does PSP source tag dir not exist??? " + pspSourceTagDir);
2052       }
2053     }
2054     
2055     //get ant and maven
2056     this.downloadAndUnzipAnt();
2057     this.downloadAndUnzipMaven();
2058 
2059     if (this.appToUpgrade == AppToUpgrade.API) {
2060       //have to build client first
2061       this.buildClient(new File(sourceDir + File.separator + "grouper-misc" + File.separator + "grouperClient"));
2062       this.buildClient(new File(sourceTagDir + File.separator + "grouper-misc" + File.separator + "grouperClient"));
2063       
2064     }
2065 
2066     // Grouper API has always been built for everything, but its build is actually broken
2067     // right now, and, in fact, there is no need to build Grouper API for PSPNG
2068     // Therefore: Disabling Grouper API build when building PSPNG so it can at least be patched
2069     if (this.appToUpgrade != AppToUpgrade.PSPNG) {
2070       this.untarredApiDir = new File(sourceDir + File.separator + "grouper");
2071       this.buildGrouperApi(new File(sourceDir + File.separator + "grouper"));
2072       this.untarredApiDir = new File(sourceTagDir + File.separator + "grouper");
2073       this.buildGrouperApi(new File(sourceTagDir + File.separator + "grouper"));
2074     }
2075     
2076     if (this.appToUpgrade == AppToUpgrade.API) {
2077       //other packages, e.g. duo, box
2078 
2079       this.buildDuo(new File(sourceDir + File.separator + "grouper-misc" + File.separator + "grouper-duo"));
2080       this.buildDuo(new File(sourceTagDir + File.separator + "grouper-misc" + File.separator + "grouper-duo"));
2081       
2082       this.buildBox(new File(sourceDir + File.separator + "grouper-misc" + File.separator + "grouper-box"));
2083       this.buildBox(new File(sourceTagDir + File.separator + "grouper-misc" + File.separator + "grouper-box"));
2084       
2085     }
2086     
2087 
2088     if (this.appToUpgrade == AppToUpgrade.UI) {
2089       //lets build the UI
2090       this.untarredApiDir = new File(sourceDir + File.separator + "grouper");
2091       this.untarredUiDir = new File(sourceDir + File.separator + "grouper-ui");
2092       this.configureUi();
2093       this.buildUi(false);
2094 
2095       this.untarredApiDir = new File(sourceTagDir + File.separator + "grouper");
2096       this.untarredUiDir = new File(sourceTagDir + File.separator + "grouper-ui");
2097       this.configureUi();
2098       this.buildUi(false);
2099 
2100     }
2101     
2102     if (this.appToUpgrade == AppToUpgrade.WS) {
2103       //lets build the WS
2104       this.untarredApiDir = new File(sourceDir + File.separator + "grouper");
2105       this.untarredWsDir = new File(sourceDir + File.separator + "grouper-ws");
2106       this.configureWs();
2107       this.buildWs(false);
2108 
2109       this.untarredApiDir = new File(sourceTagDir + File.separator + "grouper");
2110       this.untarredWsDir = new File(sourceTagDir + File.separator + "grouper-ws");
2111       this.configureWs();
2112       this.buildWs(false);
2113     }    
2114     
2115     if (this.appToUpgrade == AppToUpgrade.PSPNG) {
2116       this.untarredPspngDir = new File(sourceDir + File.separator + "grouper-misc" + File.separator + "grouper-pspng");
2117       this.buildPspng(this.untarredPspngDir);
2118 
2119       this.untarredPspngDir = new File(sourceTagDir + File.separator + "grouper-misc" + File.separator + "grouper-pspng");
2120       this.buildPspng(this.untarredPspngDir);
2121     }    
2122 
2123     if (this.appToUpgrade == AppToUpgrade.PSP) {
2124       this.untarredApiDir = new File(sourceDir + File.separator + "grouper");
2125       this.buildPsp(pspSourceDir);
2126 
2127       this.untarredApiDir = new File(sourceTagDir + File.separator + "grouper");
2128       this.buildPsp(pspSourceTagDir);
2129     }    
2130 
2131     //lets index files
2132     Map<String, GrouperInstallerIndexFile> indexOfFiles = new TreeMap<String, GrouperInstallerIndexFile>();
2133     Map<String, GrouperInstallerIndexFile> indexOfTagFiles = new TreeMap<String, GrouperInstallerIndexFile>();
2134 
2135     patchCreateIndexFiles(indexOfFiles, sourceDir, pspSourceDir);
2136     patchCreateIndexFiles(indexOfTagFiles, sourceTagDir, pspSourceTagDir);
2137 
2138     Set<GrouperInstallerIndexFile> grouperInstallerIndexFilesToAddToPatch = new HashSet<GrouperInstallerIndexFile>();
2139     
2140     //lets get the files from the user
2141     OUTER: for (int i=0;i<10;i++) {
2142       
2143       if (i==9) {
2144         throw new RuntimeException("You need to enter valid files!");
2145       }
2146 
2147       //if subsequent pass, then start fresh
2148       grouperInstallerIndexFilesToAddToPatch.clear();
2149       
2150       System.out.println("\nThe following could be filename if no dupes: Something.java.\n"
2151           + "Could be package path: edu/whatever/Something.java\n"
2152           + "could be path in module: dist/build/edu/internet2/middleware/grouper/changeLog/esb/consumer/EsbEvent.java\n"
2153           + "could be: webapp/WEB-INF/grouperUi2/index/index.jsp");
2154       System.out.println("Enter the comma separated list of files (dont use .class, use .java) to make a patch from: [required]\n");
2155       String filesToMakePatchFromCommaSeparated = readFromStdIn("grouperInstaller.autorun.patchFilesCommaSeparated");
2156       if (GrouperInstallerUtils.isBlank(filesToMakePatchFromCommaSeparated)) {
2157         System.out.println("This is a required field!");
2158         continue;
2159       }
2160       
2161       Set<String> fileKeys = new LinkedHashSet<String>(GrouperInstallerUtils.nonNull(
2162           GrouperInstallerUtils.splitTrimToList(filesToMakePatchFromCommaSeparated, ",")));
2163 
2164       for (String fileKey : fileKeys) {
2165         
2166         if (fileKey.endsWith(".class")) {
2167           System.out.println("Do not specify .class files, only .java files (will be compiled): '" + fileKey + "'!!!  please re-enter the list");
2168           continue OUTER;
2169         }
2170 
2171         GrouperInstallerIndexFile grouperInstallerIndexFile = indexOfFiles.get(fileKey);
2172         if (grouperInstallerIndexFile == null) {
2173           grouperInstallerIndexFile = indexOfTagFiles.get(fileKey);
2174           //see if we are deleting
2175           if (grouperInstallerIndexFile == null) {
2176             System.out.println("Cant find file: '" + fileKey + "'!!!  please re-enter the list");
2177             continue OUTER;
2178           }
2179         }
2180         
2181         if (grouperInstallerIndexFile.isHasMultipleFilesBySimpleName()
2182             && GrouperInstallerUtils.equals(fileKey, grouperInstallerIndexFile.getSimpleName())) {
2183           System.out.println("This name is in the index multiple times, please be more specific: '" 
2184               + fileKey + "', " + grouperInstallerIndexFile.getErrors());
2185           continue OUTER;
2186         }
2187         
2188         if (grouperInstallerIndexFile.isHasMultipleFilesByRelativePath()
2189             && GrouperInstallerUtils.equals(fileKey, grouperInstallerIndexFile.getRelativePath())) {
2190           System.out.println("This relative path is in the index multiple times, please be more specific: '" 
2191               + fileKey + "', " + grouperInstallerIndexFile.getErrors());
2192           continue OUTER;
2193         }
2194 
2195         if (grouperInstallerIndexFile.isHasMultipleFilesByPath()
2196             && GrouperInstallerUtils.equals(fileKey, grouperInstallerIndexFile.getPath())) {
2197           System.out.println("This path is in the index multiple times, please be more specific: '" 
2198               + fileKey + "', " + grouperInstallerIndexFile.getErrors());
2199           continue OUTER;
2200         }
2201         
2202         grouperInstallerIndexFilesToAddToPatch.add(grouperInstallerIndexFile);
2203       }
2204       break OUTER;
2205     }
2206     
2207     //ok, we have our list of files
2208     //lets go from java to class
2209     for (GrouperInstallerIndexFile grouperInstallerIndexFile : new HashSet<GrouperInstallerIndexFile>(grouperInstallerIndexFilesToAddToPatch)) {
2210 
2211       if (grouperInstallerIndexFile.getSimpleName().endsWith(".java")) {
2212         
2213         String relativePathJava = grouperInstallerIndexFile.getRelativePath();
2214         String relativePathPrefix = GrouperInstallerUtils.substringBeforeLast(relativePathJava, ".");
2215         String relativePathClass = relativePathPrefix + ".class";
2216 
2217         GrouperInstallerIndexFile grouperInstallerIndexFileClassFile = indexOfFiles.get(relativePathClass);
2218         
2219         //this will happen in a delete
2220         if (grouperInstallerIndexFileClassFile == null) {
2221           continue;
2222         }
2223         
2224         //this shouldnt happen
2225         if (grouperInstallerIndexFileClassFile.isHasMultipleFilesByRelativePath()) {
2226           throw new RuntimeException("Class file has multiple files by relative path???? " + relativePathClass);
2227         }
2228 
2229         //found class file
2230         grouperInstallerIndexFilesToAddToPatch.add(grouperInstallerIndexFileClassFile);
2231         
2232         //lets get all the inner classes
2233         File parentFile = grouperInstallerIndexFileClassFile.getFile().getParentFile();
2234         
2235         //with slash if needed, not sure why a class wouldnt have a package, but handle the case anyways
2236         String parentRelativePathWithSlash = GrouperInstallerUtils.substringBeforeLast(grouperInstallerIndexFileClassFile.getRelativePath(), "/") + "/";
2237         if (!grouperInstallerIndexFileClassFile.getRelativePath().contains("/")) {
2238           parentRelativePathWithSlash = "";
2239         }
2240         String fileNameInnerClassPrefix = GrouperInstallerUtils.substringBeforeLast(
2241             grouperInstallerIndexFileClassFile.getFile().getName(), ".") + "$";
2242         for (File siblingFile : parentFile.listFiles()) {
2243           if (siblingFile.getName().endsWith(".class") && GrouperInstallerUtils.filePathStartsWith(siblingFile.getName(),fileNameInnerClassPrefix)) {
2244             //this is an inner class
2245             String innerClassRelativePath = parentRelativePathWithSlash + siblingFile.getName();
2246             GrouperInstallerIndexFile innerClassIndexFile = indexOfFiles.get(innerClassRelativePath);
2247             if (innerClassIndexFile == null) {
2248               throw new RuntimeException("Cant find inner class index file??? " + innerClassRelativePath);
2249             }
2250             if (innerClassIndexFile.isHasMultipleFilesByRelativePath()) {
2251               throw new RuntimeException("Inner class file has multiple files by relative path??? " + innerClassRelativePath);
2252             }
2253             //found class file
2254             grouperInstallerIndexFilesToAddToPatch.add(innerClassIndexFile);
2255           }
2256         }
2257       }
2258     }
2259 
2260     File patchDir = new File(this.grouperTarballDirectoryString + "patches" + File.separator + patchName);
2261     
2262     if (patchDir.exists()) {
2263       if (patchDir.isFile()) {
2264         throw new RuntimeException("Why is patch directory a file???? " + patchDir.getAbsolutePath());
2265       }
2266       
2267       System.out.println("Local patch dir exists, is it ok to be automatically deleted? (t|f)? [t]:");
2268       boolean continueOn = readFromStdInBoolean(true, "grouperInstaller.autorun.deleteLocalPatchFile");
2269       if (!continueOn) {
2270         System.out.println("Cant continue if not deleting patch dir: " + patchDir.getAbsolutePath());
2271         throw new RuntimeException("Cant continue if not deleting patch dir: " + patchDir.getAbsolutePath());
2272       }
2273       
2274       //delete this dir
2275       GrouperInstallerUtils.deleteRecursiveDirectory(patchDir.getAbsolutePath());
2276       
2277     }
2278 
2279     
2280     //lets look for dependencies
2281     Set<String> dependencyPatchNames = new TreeSet<String>();
2282     
2283     //keep track of files to put in the "old" dir
2284     Map<GrouperInstallerIndexFile, File> indexFileToOldFile = new HashMap<GrouperInstallerIndexFile, File>();
2285     
2286     //go from most recent to oldest
2287     for (int i=nextPatchIndex-1;i>=0;i--) {
2288       
2289       //lets find the patch dir
2290       String currentPatchName = GrouperInstallerUtils.substringBeforeLast(patchName, "_") + "_" + i;
2291       
2292       File currentPatchDir = new File(this.grouperTarballDirectoryString + "patches" + File.separator + currentPatchName);
2293 
2294       Iterator<GrouperInstallerIndexFile> iterator = grouperInstallerIndexFilesToAddToPatch.iterator();
2295       
2296       while (iterator.hasNext()) {
2297         
2298         GrouperInstallerIndexFile indexFileToAdd = iterator.next();
2299         
2300         //dont check twice
2301         if (indexFileToOldFile.containsKey(indexFileToAdd)) {
2302           continue;
2303         }
2304         
2305         GrouperInstallerIndexFile indexFileToAddFromBranch = indexOfFiles.get(indexFileToAdd.getRelativePath());
2306         
2307         //note the old file will be in the patch's new directory
2308         File oldFile = new File(currentPatchDir.getAbsolutePath() + File.separator 
2309             + "new" + File.separator + indexFileToAdd.getPatchFileType().getDirName()
2310             + File.separator + GrouperInstallerUtils.replace(indexFileToAdd.getRelativePath(), "/", File.separator));
2311         if (oldFile.exists() && oldFile.isFile()) {
2312 
2313           if (indexFileToAddFromBranch != null && GrouperInstallerUtils.contentEquals(indexFileToAdd.getFile(), oldFile)) {
2314             System.out.println("New file is same as old file: " + indexFileToAdd.getFile().getAbsolutePath() + ", " 
2315                 + oldFile.getAbsolutePath());
2316             System.out.println("This file will not be included in patch");
2317             //remove from patch
2318             iterator.remove();
2319           } else {
2320 
2321             //this is now a dependency
2322             dependencyPatchNames.add(currentPatchName);
2323             
2324             //link this with the installer index file
2325             indexFileToOldFile.put(indexFileToAdd, oldFile);
2326           }          
2327         }
2328         
2329       }
2330       
2331     }
2332     
2333     {
2334       String patchNameDependenciesString = null;
2335       
2336       OUTER: for (int i=0;i<10;i++) {
2337         if (i==9) {
2338           throw new RuntimeException("Invalid patch names!");
2339         }
2340         if (dependencyPatchNames.size() == 0) {
2341           
2342           System.out.println("No dependency patches are detected, enter any patch names that are "
2343               + "dependencies that you know of (comma separated), or blank for none:\n");
2344           patchNameDependenciesString = readFromStdIn("grouperInstaller.autorun.patchNameDependenciesCommaSeparated");
2345           
2346         } else {
2347     
2348           System.out.println("These " + dependencyPatchNames.size() + " patches are detected: " 
2349               + GrouperInstallerUtils.join(dependencyPatchNames.iterator(), ", "));
2350           System.out.println("Enter any patch names that are dependencies that you know of (comma separated), or blank for none:\n");
2351           patchNameDependenciesString = readFromStdIn("grouperInstaller.autorun.patchNameDependenciesCommaSeparated");
2352     
2353         }
2354         if (!GrouperInstallerUtils.isBlank(patchNameDependenciesString)) {
2355           List<String> patchNameDependeciesFromUser = GrouperInstallerUtils.splitTrimToList(patchNameDependenciesString, ",");
2356           for (String currentPatchName : patchNameDependeciesFromUser) {
2357             if (!patchNameValid(currentPatchName)) {
2358               System.out.println("Invalid patch name! '" + currentPatchName + "', enter them again!");
2359               continue OUTER;
2360             }
2361           }
2362           dependencyPatchNames.addAll(patchNameDependeciesFromUser);
2363         }
2364         break;
2365       }
2366       
2367     }    
2368 
2369     //find old files from the tag
2370     Iterator<GrouperInstallerIndexFile> iterator = grouperInstallerIndexFilesToAddToPatch.iterator();
2371     
2372     while (iterator.hasNext()) {
2373       GrouperInstallerIndexFile currentIndexFile = iterator.next();
2374       //see if its covered in another patch
2375       if (indexFileToOldFile.containsKey(currentIndexFile)) {
2376         continue;
2377       }
2378       
2379       //dont have old files from java or classes, thats only for other patches to do
2380       if (currentIndexFile.getSimpleName().endsWith(".class") || currentIndexFile.getSimpleName().endsWith(".java")) {
2381         continue;
2382       }
2383 
2384       GrouperInstallerIndexFile currentIndexFileFromBranch = indexOfFiles.get(currentIndexFile.getRelativePath());
2385       
2386       //look for the old file
2387       GrouperInstallerIndexFile currentIndexFileFromTag = indexOfTagFiles.get(currentIndexFile.getPath());
2388       if (currentIndexFileFromTag == null) {
2389         currentIndexFileFromTag = indexOfTagFiles.get(currentIndexFile.getRelativePath());
2390       }
2391       if (currentIndexFileFromTag != null) {
2392         if (currentIndexFileFromTag.isHasMultipleFilesByPath()) {
2393           throw new RuntimeException("Why multiple paths???? " + currentIndexFile + ", " + currentIndexFile.getPath());
2394         }
2395         if (currentIndexFileFromBranch != null && GrouperInstallerUtils.contentEquals(currentIndexFileFromTag.getFile(), currentIndexFile.getFile())) {
2396           System.out.println("New file is same as old file: " + currentIndexFile.getFile().getAbsolutePath() + ", " 
2397               + currentIndexFileFromTag.getFile().getAbsolutePath());
2398           System.out.println("This file will not be included in patch");
2399           //remove from patch
2400           iterator.remove();
2401         } else {
2402           //add this as an old file
2403           indexFileToOldFile.put(currentIndexFile, currentIndexFileFromTag.getFile());
2404         }
2405       }
2406     }
2407     
2408     if (grouperInstallerIndexFilesToAddToPatch.size() == 0) {
2409       throw new RuntimeException("There are no files to put in patch!");
2410     }
2411     
2412 
2413     //# will show up on screen so user knows what it is
2414     //description = This patch fixes GRP-1080: browse folders refresh button only works in chrome, not other browsers
2415     System.out.print("\nEnter a description for this patch, e.g. GRP-123: fixes a problem with such and such: [required]\n");
2416     String patchDescription = readFromStdIn("grouperInstaller.autorun.patchDescription");
2417     
2418     if (GrouperInstallerUtils.isBlank(patchDescription)) {
2419       throw new RuntimeException("Cant have a blank description!");
2420     }
2421 
2422     //# (note, will try to get this from patch description, if its there, this can be blank)
2423     Matcher patchJiraKeyMatcher = Pattern.compile(".*(GRP-\\d+).*").matcher(patchDescription);
2424     String defaultPatchJiraKey = "";
2425     if (patchJiraKeyMatcher.matches()) {
2426       defaultPatchJiraKey = patchJiraKeyMatcher.group(1);
2427     }
2428     System.out.print("\nEnter a Jira key (e.g. GRP-123) for this patch: [required] " 
2429         + (GrouperInstallerUtils.isBlank(defaultPatchJiraKey) ? "" : ("[" + defaultPatchJiraKey + "]")) + "\n");
2430     String patchJiraKey = readFromStdIn("grouperInstaller.autorun.patchJiraKey");
2431     
2432     if (GrouperInstallerUtils.isBlank(patchJiraKey)) {
2433       if (!GrouperInstallerUtils.isBlank(defaultPatchJiraKey)) {
2434         patchJiraKey = defaultPatchJiraKey;
2435       } else {
2436         throw new RuntimeException("Cant have a blank jira key!");
2437       }
2438     }
2439     
2440     if (!Pattern.compile("^GRP-\\d+$").matcher(patchJiraKey).matches()) {
2441       throw new RuntimeException("Patch jira key must be valid: '" + patchJiraKey + "'");
2442     }
2443 
2444     String patchRiskLevel = null;
2445     
2446     {
2447       //# low, medium, or high risk to applying the patch
2448       //risk = low
2449       System.out.println("Enter the risk level for the patch: (low|medium|high): [required] ");
2450       String patchRiskLevelInput = readFromStdIn("grouperInstaller.autorun.patchRiskLevel");
2451       
2452       if (GrouperInstallerUtils.equalsIgnoreCase("low", patchRiskLevelInput)) {
2453         patchRiskLevel = "low";
2454       } else if (GrouperInstallerUtils.equalsIgnoreCase("medium", patchRiskLevelInput)) {
2455         patchRiskLevel = "medium";
2456       } else if (GrouperInstallerUtils.equalsIgnoreCase("high", patchRiskLevelInput)) {
2457         patchRiskLevel = "high";
2458       } else {
2459         throw new RuntimeException("Invalid risk level: '" + patchRiskLevelInput + "', expecting low|medium|high");
2460       }
2461       
2462     }
2463     
2464     //# is this is a security patch (true or false)
2465     //security = false
2466     System.out.println("Is this a security patch? (t|f): [t] ");
2467     boolean securityPatch = readFromStdInBoolean(false, "grouperInstaller.autorun.patchSecurity");
2468 
2469     boolean requiresRestart = false;
2470     for (GrouperInstallerIndexFile currentIndexFile : grouperInstallerIndexFilesToAddToPatch) {
2471       if (currentIndexFile.getSimpleName().endsWith(".jar")
2472           || currentIndexFile.getSimpleName().endsWith(".java")) {
2473         requiresRestart = true;
2474       }
2475     }
2476     //# if this patch requires restart of processes (true or false)
2477     //requiresRestart = false
2478     if (requiresRestart) {
2479       System.out.println("It is detected that your patch requires restart");
2480     } else {
2481       System.out.println("It is NOT detected that your patch requires restart, please confirm this, does it require restart (t|f)? [f] ");
2482       requiresRestart = readFromStdInBoolean(false, "grouperInstaller.autorun.overrideDoesntRequireRestart");
2483       
2484       if (requiresRestart) {
2485         System.out.println("Perhaps the maintainer of the Grouper Installer can use this feedback to make a better guess on restart, let them know");
2486         GrouperInstallerUtils.sleep(2000);
2487       }
2488     }
2489     //at this point we can build the patch dir and file and put files in there
2490     
2491     //# patches that this patch is dependant on (comma separated)
2492     //dependencies = 
2493 
2494     //create the dir
2495     GrouperInstallerUtils.mkdirs(patchDir);
2496     
2497     {
2498       String patchPropertiesContents = "# will show up on screen so user knows what it is\n"
2499           + "description = " + patchDescription + "\n"
2500           + "\n"
2501           + "# patches that this patch is dependant on (comma separated)\n"
2502           + "dependencies = " + GrouperInstallerUtils.join(dependencyPatchNames.iterator(), ", ") + "\n"
2503           + "\n"
2504           + "# low, medium, or high risk to applying the patch\n"
2505           + "risk = " + patchRiskLevel + "\n"
2506           + "\n"
2507           + "# is this is a security patch (true or false)\n"
2508           + "security = " + securityPatch + "\n"
2509           + "\n"
2510           + "# if this patch requires restart of processes (true or false)\n"
2511           + "requiresRestart = " + requiresRestart + "\n";
2512       String patchPropertiesFileName = patchDir + File.separator + patchDir.getName() + ".properties";
2513       GrouperInstallerUtils.saveStringIntoFile(new File(patchPropertiesFileName), patchPropertiesContents);
2514     }
2515     
2516     //lets do old files
2517     //start with old files
2518     if (indexFileToOldFile.size() > 0) {
2519       GrouperInstallerUtils.mkdirs(new File(patchDir.getAbsolutePath() + File.separator + "old"));
2520       for (GrouperInstallerIndexFile currentIndexFile : indexFileToOldFile.keySet()) {
2521 
2522         File oldFile = new File(patchDir.getAbsolutePath() + File.separator + "old" + File.separator
2523             + currentIndexFile.getPatchFileType().getDirName() + File.separator
2524             + GrouperInstallerUtils.replace(currentIndexFile.getRelativePath(), "/", File.separator));
2525         
2526         GrouperInstallerUtils.mkdirs(oldFile.getParentFile());
2527         
2528         System.out.println("Copying old file from " + indexFileToOldFile.get(currentIndexFile).getAbsolutePath()
2529             + "\n   to: " + oldFile.getAbsolutePath());
2530         
2531         GrouperInstallerUtils.copyFile(indexFileToOldFile.get(currentIndexFile), oldFile);
2532         
2533       }
2534     }
2535 
2536     //now put new files in place
2537     {
2538       GrouperInstallerUtils.mkdirs(new File(patchDir.getAbsolutePath() + File.separator + "new"));
2539       for (GrouperInstallerIndexFile currentIndexFile : grouperInstallerIndexFilesToAddToPatch) {
2540 
2541         //is it a delete? then there is no new file
2542         if (!indexOfFiles.containsKey(currentIndexFile.getRelativePath())) {
2543           continue;
2544         }
2545         
2546         File newFile = new File(patchDir.getAbsolutePath() + File.separator + "new" + File.separator
2547             + currentIndexFile.getPatchFileType().getDirName() + File.separator
2548             + GrouperInstallerUtils.replace(currentIndexFile.getRelativePath(), "/", File.separator));
2549         
2550         GrouperInstallerUtils.mkdirs(newFile.getParentFile());
2551         
2552         System.out.println("Copying new file from " + currentIndexFile.getFile().getAbsolutePath()
2553             + "\n   to: " + newFile.getAbsolutePath());
2554         
2555         GrouperInstallerUtils.copyFile(currentIndexFile.getFile().getAbsoluteFile(), newFile);
2556         
2557       }
2558     }
2559 
2560     {
2561       //generate the wiki markup
2562       //    <tr>
2563       //      <td>
2564       //        <p>
2565       //          <a href="https://software.internet2.edu/grouper/release/2.2.1/patches/grouper_v2_2_1_api_patch_1.tar.gz">grouper_v2_2_1_api_patch_1</a>
2566       //        </p>
2567       //      </td>
2568       //      <td>
2569       //        <p>
2570       //          <a href="https://bugs.internet2.edu/jira/browse/GRP-1096">GRP-1096: Use threads for 2.2 upgrade to decrease time of upgrade</a>
2571       //        </p>
2572       //      </td>
2573       //      <td>
2574       //        <p>classes/edu/internet2/middleware/grouper/internal/dao/hib3/Hib3StemSetDAO.java <br class="atl-forced-newline"/> classes/edu/internet2/middleware/grouper/misc/MigrateLegacyAttributes.java <br class="atl-forced-newline"/> classes/edu/internet2/middleware/grouper/misc/AddMissingGroupSets.java <br class="atl-forced-newline"/> classes/edu/internet2/middleware/grouper/misc/SyncPITTables.java <br class="atl-forced-newline"/> classes/edu/internet2/middleware/grouper/misc/SyncStemSets.java <br class="atl-forced-newline"/> classes/grouper.base.properties</p>
2575       //      </td>
2576       //    </tr>
2577       String grouperVersion = GrouperInstallerUtils.propertiesValue("grouper.version", true);
2578 
2579       String wikiMarkup = "    <tr>\n"
2580         + "      <td>\n"
2581         + "        <p>" + new SimpleDateFormat("yyyy/MM/dd").format(new Date()) + "</p>\n"
2582         + "      </td>\n"
2583         + "      <td>\n"
2584         + "        <p>\n"
2585         + "          <a href=\"https://software.internet2.edu/grouper/release/" + grouperVersion + "/patches/" + patchName + ".tar.gz\">" + patchName + "</a>\n"
2586         + "        </p>\n"
2587         + "      </td>\n"
2588         + "      <td>\n"
2589         + "        <p>\n"
2590         + "          <a href=\"https://bugs.internet2.edu/jira/browse/" + patchJiraKey + "\">" + patchDescription + "</a>\n"
2591         + "        </p>\n"
2592         + "      </td>\n"
2593         + "      <td>\n"
2594         + "        <p>";
2595       
2596       boolean isFirst = true;
2597       for (GrouperInstallerIndexFile currentIndexFile : grouperInstallerIndexFilesToAddToPatch) {
2598         
2599         //just do java files
2600         if (currentIndexFile.getSimpleName().endsWith(".class")) {
2601           continue;
2602         }
2603         
2604         //classes/edu/internet2/middleware/grouper/internal/dao/hib3/Hib3StemSetDAO.java 
2605         //<br class=\"atl-forced-newline\"/>
2606 
2607         if (!isFirst) {
2608           wikiMarkup += "<br class=\"atl-forced-newline\"/>";
2609         }
2610         wikiMarkup += currentIndexFile.getPatchFileType().getDirName() + "/" 
2611             + currentIndexFile.getRelativePath();
2612 
2613         isFirst = false;
2614         
2615       }
2616       wikiMarkup += "</p>\n      </td>\n"
2617         + "    </tr>\n";
2618     
2619       System.out.println("Here is the wiki markup for the release notes page, copy and paste that into confluence using the <> button:");
2620       System.out.println("\n" + wikiMarkup + "\n");
2621       System.out.print("Press <enter> to continue... ");
2622       readFromStdIn("grouperInstaller.autorun.patchContinueAfterWikiMarkup");
2623     }
2624     
2625     // tar this up
2626     File tarfile = new File(patchDir.getParentFile() + File.separator + patchName + ".tar");
2627     GrouperInstallerUtils.tar(patchDir, tarfile);
2628     
2629     System.out.println("\nDo you want to name this file as a test version so you can test it without affecting other users? (t|f) [t]: ");
2630     boolean patchUseTestFileName = readFromStdInBoolean(true, "grouperInstaller.autorun.patchNameFileAsTestVersion");
2631 
2632     File gzipfile = new File(patchDir.getParentFile() + File.separator + patchName + (patchUseTestFileName ? "_test" : "") + ".tar.gz");
2633     GrouperInstallerUtils.gzip(tarfile, gzipfile);
2634 
2635     System.out.println("\nSUCCESS: your patch is here: " + gzipfile.getAbsolutePath());
2636 
2637   }
2638 
2639   /**
2640    * patch pattern
2641    */
2642   private static final Pattern patchNamePattern = Pattern.compile("^grouper_v(\\d+)_(\\d+)_(\\d+)_(api|ws|ui|psp|pspng)_patch_(\\d+)$");
2643 
2644   
2645   /**
2646    * see if valid patch name e.g. grouper_v2_2_1_api_patch_0
2647    * @param patchName
2648    * @return true for valid
2649    */
2650   private static boolean patchNameValid(String patchName) {
2651     //validate patch names
2652     return patchNamePattern.matcher(patchName).matches();
2653 
2654   }
2655   
2656   /**
2657    * index files from a source directory
2658    * @param theIndexOfFiles index of label to the index file object
2659    * @param theSourceDir to look for files in
2660    * @param thePspSourceDir is psp source dir to look for files in
2661    */
2662   private void patchCreateIndexFiles(Map<String, GrouperInstallerIndexFile> theIndexOfFiles, File theSourceDir, File thePspSourceDir) {
2663     System.out.println("\nCreating file index to make patches from " + theSourceDir.getAbsolutePath() + "...\n");
2664     
2665     switch(this.appToUpgrade) {
2666       case CLIENT:
2667         throw new RuntimeException("No patching client, patch API instead");
2668       case API:
2669 
2670         //index the grouper client
2671 // dont think we need lib from client, only api
2672 //        this.patchCreateProcessFiles(indexOfFiles, 
2673 //            new File(sourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouperClient"),
2674 //            new File(sourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouperClient" + File.separator + "lib"),
2675 //            PatchFileType.lib);
2676 
2677         this.patchCreateProcessFiles(theIndexOfFiles, 
2678             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouperClient"),
2679             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouperClient" 
2680                 + File.separator + "dist" + File.separator + "bin"),
2681             PatchFileType.clazz);
2682 
2683         this.patchCreateProcessFiles(theIndexOfFiles, 
2684             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouperClient"),
2685             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouperClient" 
2686                 + File.separator + "src" + File.separator + "java"),
2687             PatchFileType.clazz);
2688 
2689         this.patchCreateProcessFiles(theIndexOfFiles, 
2690             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouperClient"),
2691             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouperClient" 
2692                 + File.separator + "src" + File.separator + "ext"),
2693             PatchFileType.clazz);
2694 
2695         this.patchCreateProcessFiles(theIndexOfFiles, 
2696             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouperClient"),
2697             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouperClient" 
2698                 + File.separator + "conf"),
2699             PatchFileType.clazz);
2700 
2701         // duo
2702         this.patchCreateProcessFiles(theIndexOfFiles, 
2703             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-duo"),
2704             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-duo" 
2705                 + File.separator + "src"),
2706             PatchFileType.clazz);
2707 
2708         this.patchCreateProcessFiles(theIndexOfFiles, 
2709             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-duo"),
2710             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-duo" 
2711                 + File.separator + "dist" + File.separator + "bin"),
2712             PatchFileType.clazz);
2713 
2714         // box
2715         this.patchCreateProcessFiles(theIndexOfFiles, 
2716             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-box"),
2717             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-box" 
2718                 + File.separator + "src"),
2719             PatchFileType.clazz);
2720 
2721         this.patchCreateProcessFiles(theIndexOfFiles, 
2722             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-box"),
2723             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-box" 
2724                 + File.separator + "changeLogConsumerSource"),
2725             PatchFileType.clazz);
2726 
2727         this.patchCreateProcessFiles(theIndexOfFiles, 
2728             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-box"),
2729             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-box" 
2730                 + File.separator + "dist" + File.separator + "bin"),
2731             PatchFileType.clazz);
2732 
2733         //add grouper api files
2734         this.patchCreateProcessFiles(theIndexOfFiles, 
2735             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper"),
2736             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper" + File.separator + "lib"),
2737             PatchFileType.lib);
2738 
2739         this.patchCreateProcessFiles(theIndexOfFiles, 
2740             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper"),
2741             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper" + File.separator + "dist" 
2742                 + File.separator + "build" + File.separator + "grouper"),
2743             PatchFileType.clazz);
2744 
2745         this.patchCreateProcessFiles(theIndexOfFiles,
2746             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper"),
2747             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper" + File.separator + "conf"),
2748             PatchFileType.clazz);
2749 
2750         this.patchCreateProcessFiles(theIndexOfFiles,
2751             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper"),
2752             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper" + File.separator + "src" 
2753                 + File.separator + "grouper"),
2754             PatchFileType.clazz);
2755 
2756         this.patchCreateProcessFiles(theIndexOfFiles,
2757             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper"),
2758             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper" + File.separator + "src" 
2759                 + File.separator + "esb"),
2760             PatchFileType.clazz);
2761 
2762 // do this at some point
2763 //        this.patchCreateProcessFiles(theIndexOfFiles,
2764 //            new File(theSourceDir.getAbsolutePath() + File.separator + "grouper"),
2765 //            new File(theSourceDir.getAbsolutePath() + File.separator + "grouper" + File.separator + "src" 
2766 //                + File.separator + "test"),
2767 //            PatchFileType.clazz);
2768 
2769         this.patchCreateProcessFiles(theIndexOfFiles,
2770             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper"),
2771             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper" + File.separator + "bin"),
2772             PatchFileType.bin);
2773 
2774 
2775         break;
2776       case UI:
2777         
2778         this.patchCreateProcessFiles(theIndexOfFiles, 
2779             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ui"),
2780             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ui" 
2781                 + File.separator + "java" + File.separator + "lib"),
2782             PatchFileType.lib);
2783         
2784         this.patchCreateProcessFiles(theIndexOfFiles, 
2785             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ui"),
2786             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ui" 
2787                 + File.separator + "java" + File.separator + "src"),
2788             PatchFileType.clazz);
2789         
2790         this.patchCreateProcessFiles(theIndexOfFiles, 
2791             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ui"),
2792             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ui" 
2793                 + File.separator + "conf"),
2794             PatchFileType.clazz);
2795         
2796         this.patchCreateProcessFiles(theIndexOfFiles, 
2797             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ui"),
2798             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ui" 
2799                 + File.separator + "temp" + File.separator + "jarBin"),
2800             PatchFileType.clazz);
2801         
2802         this.patchCreateProcessFiles(theIndexOfFiles, 
2803             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ui"),
2804             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ui" 
2805                 + File.separator + "webapp"),
2806             PatchFileType.file);
2807 
2808         break;
2809       case WS:
2810         
2811         this.patchCreateProcessFiles(theIndexOfFiles, 
2812             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator + "grouper-ws"),
2813             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator + "grouper-ws" 
2814                 + File.separator + "lib" + File.separator + "grouper-ws"),
2815             PatchFileType.lib);
2816 
2817         this.patchCreateProcessFiles(theIndexOfFiles, 
2818             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator + "grouper-ws"),
2819             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator + "grouper-ws" 
2820                 + File.separator + "lib" + File.separator + "rampart"),
2821             PatchFileType.lib);
2822 
2823         this.patchCreateProcessFiles(theIndexOfFiles, 
2824             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator + "grouper-ws"),
2825             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator + "grouper-ws" 
2826                 + File.separator + "build" + File.separator + "grouper-ws"),
2827             PatchFileType.clazz);
2828         
2829         this.patchCreateProcessFiles(theIndexOfFiles, 
2830             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator + "grouper-ws"),
2831             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator + "grouper-ws" 
2832                 + File.separator + "conf"),
2833             PatchFileType.clazz);
2834         
2835         // we need to get all the source folders except test, note, each release adds another
2836         File parentSourceDir = new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator + "grouper-ws"
2837             + File.separator + "src");
2838 
2839         for (File wsSourceDir : parentSourceDir.listFiles()) {
2840           if (wsSourceDir.isFile() || !wsSourceDir.getName().startsWith("grouper")) {
2841             continue;
2842           }
2843           this.patchCreateProcessFiles(theIndexOfFiles, 
2844               new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator + "grouper-ws"),
2845               new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator + "grouper-ws" 
2846                   + File.separator + "src" + File.separator + wsSourceDir.getName()),
2847               PatchFileType.clazz);
2848         }
2849 
2850         //files
2851         this.patchCreateProcessFiles(theIndexOfFiles, 
2852             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator + "grouper-ws"),
2853             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator + "grouper-ws" 
2854                 + File.separator + "webapp"),
2855             PatchFileType.file);
2856         
2857         break;
2858         
2859       case PSP:
2860         this.patchCreateProcessFiles(theIndexOfFiles, 
2861             new File(thePspSourceDir.getAbsolutePath() + File.separator + "psp"),
2862             new File(thePspSourceDir.getAbsolutePath() + File.separator + "psp" + File.separator + "target" 
2863                 + File.separator + "dependency"),
2864             PatchFileType.lib);
2865         this.patchCreateProcessFiles(theIndexOfFiles, 
2866             new File(thePspSourceDir.getAbsolutePath() + File.separator + "psp"),
2867             new File(thePspSourceDir.getAbsolutePath() + File.separator + "psp" + File.separator + "src" 
2868                 + File.separator + "main" + File.separator + "java"),
2869             PatchFileType.clazz);
2870         this.patchCreateProcessFiles(theIndexOfFiles, 
2871             new File(thePspSourceDir.getAbsolutePath() + File.separator + "psp"),
2872             new File(thePspSourceDir.getAbsolutePath() + File.separator + "psp" + File.separator + "src" 
2873                 + File.separator + "main" + File.separator + "resources"),
2874             PatchFileType.clazz);
2875         this.patchCreateProcessFiles(theIndexOfFiles, 
2876             new File(thePspSourceDir.getAbsolutePath() + File.separator + "psp"),
2877             new File(thePspSourceDir.getAbsolutePath() + File.separator + "psp" + File.separator + "target" 
2878                 + File.separator + "classes"),
2879             PatchFileType.clazz);
2880 
2881         break;
2882       case PSPNG:
2883 
2884         this.patchCreateProcessFiles(theIndexOfFiles, 
2885             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-pspng"),
2886             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-pspng" 
2887                 + File.separator + "target" + File.separator + "dependency"),
2888             PatchFileType.lib);
2889 
2890         this.patchCreateProcessFiles(theIndexOfFiles, 
2891             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-pspng"),
2892             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-pspng" 
2893                 + File.separator + "src" + File.separator + "main" + File.separator + "java"),
2894             PatchFileType.clazz);
2895 
2896         this.patchCreateProcessFiles(theIndexOfFiles, 
2897             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-pspng"),
2898             new File(theSourceDir.getAbsolutePath() + File.separator + "grouper-misc" + File.separator + "grouper-pspng" 
2899                 + File.separator + "target" + File.separator + "classes"),
2900             PatchFileType.clazz);
2901 
2902         break;
2903     }
2904     
2905     //print out files for debugging
2906     //for (String key : theIndexOfFiles.keySet()) {
2907     //  if (key.toLowerCase().contains("mygroupsmemberships")) {
2908     //    System.out.println(key + " -> " + theIndexOfFiles.get(key));
2909     //  }
2910     //}
2911 
2912     System.out.println("\nDone creating file index to make patches from " + theSourceDir.getAbsolutePath() + "... found " + theIndexOfFiles.size() + " files\n");
2913 
2914   }
2915   
2916   /**
2917    * @param directory to look in
2918    * @param projectDirectory is the directory where the project is for the files
2919    * @param indexOfFiles the index
2920    * @param patchFileType
2921    */
2922   private void patchCreateProcessFiles(Map<String, GrouperInstallerIndexFile> indexOfFiles, File projectDirectory, File directory, 
2923       PatchFileType patchFileType) {
2924     
2925     this.patchCreateProcessFilesHelper(indexOfFiles, projectDirectory, directory, patchFileType, "");
2926 
2927   }
2928 
2929   /**
2930    * @param directory to look in
2931    * @param projectDirectory is the directory where the project is for the files
2932    * @param indexOfFiles
2933    * @param relativePath in the main path to look in, helps with restrictions
2934    * @param patchFileType
2935    */
2936   private void patchCreateProcessFilesHelper(Map<String, GrouperInstallerIndexFile> indexOfFiles, 
2937       File projectDirectory, File directory, 
2938       PatchFileType patchFileType, String relativePath) {
2939 
2940     try {
2941       //lets spider through directory and add files to index
2942       //get the files into a vector
2943       File[] allFiles = directory.listFiles();
2944   
2945       //loop through the array
2946       for (int i = 0; i < allFiles.length; i++) {
2947   
2948         File currentFileOrDirectory = allFiles[i];
2949         
2950         if (-1 < currentFileOrDirectory.getName().indexOf("..")) {
2951           continue; //dont go to the parent directory
2952         }
2953   
2954         //go to sub directory
2955         String newRelativePath = GrouperInstallerUtils.isBlank(relativePath) ? currentFileOrDirectory.getName() 
2956             : (relativePath + "/" + currentFileOrDirectory.getName());
2957   
2958         if (currentFileOrDirectory.isFile()) {
2959   
2960           boolean addFile = false;
2961           
2962           String fileRelativePath = GrouperInstallerUtils.fileRelativePath(projectDirectory, currentFileOrDirectory);
2963   
2964           switch(patchFileType) {
2965   
2966             case lib:
2967               
2968               if (currentFileOrDirectory.getName().endsWith(".jar")) {
2969                 addFile = true;
2970               }
2971   
2972               break;
2973             case file:
2974               addFile = true;
2975               
2976               if (currentFileOrDirectory.getName().endsWith(".jar")) {
2977                 addFile = false;
2978               }
2979   
2980               if (currentFileOrDirectory.getName().endsWith(".class")) {
2981                 addFile = false;
2982               }
2983   
2984               if (currentFileOrDirectory.getName().endsWith(".java")) {
2985                 addFile = false;
2986               }
2987   
2988               //these are classes not files
2989               if (GrouperInstallerUtils.filePathStartsWith(fileRelativePath,"WEB-INF/classes")) {
2990                 addFile = false;
2991               }
2992   
2993               //these are libs not files
2994               if (GrouperInstallerUtils.filePathStartsWith(fileRelativePath,"WEB-INF/lib")) {
2995                 addFile = false;
2996               }
2997   
2998               break;
2999             default: 
3000               addFile = true;
3001           }
3002   
3003           if (addFile) {
3004             GrouperInstallerIndexFilee.html#GrouperInstallerIndexFile">GrouperInstallerIndexFile grouperInstallerIndexFile = new GrouperInstallerIndexFile();
3005             grouperInstallerIndexFile.setSimpleName(currentFileOrDirectory.getName());
3006             grouperInstallerIndexFile.setRelativePath(newRelativePath);
3007             grouperInstallerIndexFile.setFile(currentFileOrDirectory);
3008             grouperInstallerIndexFile.setPatchFileType(patchFileType);
3009             grouperInstallerIndexFile.setPath(fileRelativePath);
3010             
3011             //add by name
3012             if (patchCreateAddFileToIndex(indexOfFiles, grouperInstallerIndexFile, currentFileOrDirectory.getName())) {
3013               //different file
3014               indexOfFiles.get(currentFileOrDirectory.getName()).setHasMultipleFilesBySimpleName(true);
3015               System.out.println("Note: duplicate file by name: " + currentFileOrDirectory.getAbsolutePath().replace('\\', '/') 
3016                   + ", " + currentFileOrDirectory.getName() + ", " + newRelativePath.replace('\\', '/') + ", " 
3017                   + indexOfFiles.get(currentFileOrDirectory.getName()).getRelativePath().replace('\\', '/')  + ", "
3018                   + grouperInstallerIndexFile.getPath().replace('\\', '/') + ", "
3019                   + indexOfFiles.get(currentFileOrDirectory.getName()).getPath().replace('\\', '/'));
3020             }
3021             
3022             //add by relative path
3023             if (patchCreateAddFileToIndex(indexOfFiles, grouperInstallerIndexFile, newRelativePath)) {
3024               //different file
3025               indexOfFiles.get(currentFileOrDirectory.getName()).setHasMultipleFilesByRelativePath(true);
3026               System.out.println("Note: duplicate file by relative path: " + currentFileOrDirectory.getAbsolutePath().replace('\\', '/') 
3027                   + ", " + currentFileOrDirectory.getName() + ", " + newRelativePath.replace('\\', '/') + ", " 
3028                   + indexOfFiles.get(currentFileOrDirectory.getName()).getRelativePath().replace('\\', '/')  + ", "
3029                   + grouperInstallerIndexFile.getPath().replace('\\', '/') + ", "
3030                   + indexOfFiles.get(currentFileOrDirectory.getName()).getPath().replace('\\', '/'));
3031             }
3032   
3033             //add by path
3034             if (patchCreateAddFileToIndex(indexOfFiles, grouperInstallerIndexFile, grouperInstallerIndexFile.getPath())) {
3035               //different file
3036               indexOfFiles.get(currentFileOrDirectory.getName()).setHasMultipleFilesByPath(true);
3037               System.out.println("Note: duplicate file by path: " + currentFileOrDirectory.getAbsolutePath() .replace('\\', '/') 
3038                   + ", " + currentFileOrDirectory.getName() + ", " + newRelativePath.replace('\\', '/') + ", " 
3039                   + indexOfFiles.get(currentFileOrDirectory.getName()).getRelativePath().replace('\\', '/')  + ", "
3040                   + grouperInstallerIndexFile.getPath().replace('\\', '/') + ", "
3041                   + indexOfFiles.get(currentFileOrDirectory.getName()).getPath().replace('\\', '/'));
3042             }
3043           }
3044           
3045         } else {
3046                   
3047           patchCreateProcessFilesHelper(indexOfFiles, projectDirectory, currentFileOrDirectory, patchFileType, newRelativePath);
3048           
3049         }
3050       }
3051     } catch (Exception e) {
3052       throw new RuntimeException("Problem with directory: " + directory.getAbsolutePath(), e);
3053     }
3054     
3055   }
3056 
3057   /**
3058    * @param indexOfFiles database of files by various lookup names
3059    * @param grouperInstallerIndexFile file to add
3060    * @param key add by this key
3061    * @return true if file already there and different
3062    */
3063   private boolean patchCreateAddFileToIndex(Map<String, GrouperInstallerIndexFile> indexOfFiles, 
3064       GrouperInstallerIndexFile grouperInstallerIndexFile, String key) {
3065     
3066     //convert slashes on key
3067     key = key.replace('\\', '/');
3068     
3069     grouperInstallerIndexFile.getErrors().append("Key: ").append(key).append(", ");
3070     
3071     GrouperInstallerIndexFile currentFileInIndex = indexOfFiles.get(key);
3072     if (currentFileInIndex == null) {
3073       indexOfFiles.put(key, grouperInstallerIndexFile);
3074     } else {
3075       currentFileInIndex.getErrors().append("Key: ").append(key).append(",");
3076       //skip these, who cares, too many dupes
3077       if (!GrouperInstallerUtils.equals(grouperInstallerIndexFile.getSimpleName(), "package-info.java")
3078           && !GrouperInstallerUtils.equals(grouperInstallerIndexFile.getSimpleName(), "package.html")) {
3079         if (!GrouperInstallerUtils.equals(grouperInstallerIndexFile.computeSha1(), currentFileInIndex.computeSha1())) {
3080           return true;
3081         }
3082       }
3083     }
3084     return false;
3085   }
3086   
3087   /**
3088    * build PSP
3089    * @param pspDir
3090    */
3091   private void buildPsp(File pspDir) {
3092     if (!pspDir.exists() || pspDir.isFile()) {
3093       throw new RuntimeException("Cant find psp: " + pspDir.getAbsolutePath());
3094     }
3095     
3096     File pspBuildToDir = new File(pspDir.getAbsolutePath() + File.separator + "psp" 
3097         + File.separator + "target" + File.separator + "classes");
3098     
3099     boolean rebuildPsp = true;
3100     
3101     if (pspBuildToDir.exists()) {
3102       System.out.print("The PSP has been built in the past, do you want it rebuilt? (t|f) [t]: ");
3103       rebuildPsp = readFromStdInBoolean(true, "grouperInstaller.autorun.rebuildPspAfterHavingBeenBuilt");
3104     }
3105     
3106     if (!rebuildPsp) {
3107       return;
3108     }
3109     
3110     List<String> commands = new ArrayList<String>();
3111     
3112 //    \bin\mvn compile -DskipTests
3113     addMavenCommands(commands);
3114 
3115     //put 'compile -DskipTests' in there so it wont run tests which we dont want to do
3116     // dependency:copy-dependencies package -DskipTests
3117     //not compile
3118     commands.add("dependency:copy-dependencies");
3119     commands.add("package");
3120     commands.add("-DskipTests");
3121     commands.add("-Drat.ignoreErrors=true");
3122     commands.add("-Dlicense.skip=true");
3123     
3124     System.out.println("\n##################################");
3125     System.out.println("Building PSP with command:\n" + pspDir.getAbsolutePath() + "> " 
3126         + convertCommandsIntoCommand(commands) + "\n");
3127     
3128     CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
3129         true, true, null, new File(pspDir.getAbsolutePath() + File.separator + "psp-parent"), null, true);
3130     
3131     if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
3132       System.out.println("stderr: " + commandResult.getErrorText());
3133     }
3134     if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
3135       System.out.println("stdout: " + commandResult.getOutputText());
3136     }
3137 
3138     System.out.println("\nEnd building PSP");
3139     System.out.println("##################################\n");
3140     
3141   }
3142 
3143   /**
3144    * build grouper API
3145    * @param grouperApiDir
3146    */
3147   private void buildGrouperApi(File grouperApiDir) {
3148 
3149     if (!grouperApiDir.exists() || grouperApiDir.isFile()) {
3150       throw new RuntimeException("Cant find grouper api: " + grouperApiDir.getAbsolutePath());
3151     }
3152     
3153     File grouperBuildToDir = new File(grouperApiDir.getAbsolutePath() + File.separator + "dist" + File.separator + "build" 
3154         + File.separator + "grouper");
3155     
3156     boolean rebuildGrouperApi = true;
3157     
3158     if (grouperBuildToDir.exists()) {
3159       System.out.print("The Grouper API has been built in the past, do you want it rebuilt? (t|f) [t]: ");
3160       rebuildGrouperApi = readFromStdInBoolean(true, "grouperInstaller.autorun.rebuildGrouperApiAfterHavingBeenBuilt");
3161     }
3162     
3163     if (!rebuildGrouperApi) {
3164       return;
3165     }
3166     
3167     List<String> commands = new ArrayList<String>();
3168     
3169     addAntCommands(commands);
3170 
3171     //this will run tests which we dont want to do
3172     commands.add("dist");
3173     
3174     System.out.println("\n##################################");
3175     System.out.println("Building grouper API with command:\n" + grouperApiDir.getAbsolutePath() + "> " 
3176         + convertCommandsIntoCommand(commands) + "\n");
3177     
3178     CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
3179         true, true, null, grouperApiDir, null, true);
3180     
3181     if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
3182       System.out.println("stderr: " + commandResult.getErrorText());
3183     }
3184     if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
3185       System.out.println("stdout: " + commandResult.getOutputText());
3186     }
3187 
3188     System.out.println("\nEnd building grouper API");
3189     System.out.println("##################################\n");
3190     
3191   }
3192   
3193 
3194 
3195   /**
3196    * build client API
3197    * @param clientDir
3198    */
3199   private void buildClient(File clientDir) {
3200     if (!clientDir.exists() || clientDir.isFile()) {
3201       throw new RuntimeException("Cant find client: " + clientDir.getAbsolutePath());
3202     }
3203     
3204     File clientBuildToDir = new File(clientDir.getAbsoluteFile() + File.separator + "dist" + File.separator + "bin");
3205     
3206     boolean rebuildClient = true;
3207     
3208     if (clientBuildToDir.exists()) {
3209       System.out.print("The Grouper client has been built in the past, do you want it rebuilt? (t|f) [t]: ");
3210       rebuildClient = readFromStdInBoolean(true, "grouperInstaller.autorun.rebuildClientAfterHavingBeenBuilt");
3211     }
3212     
3213     if (!rebuildClient) {
3214       return;
3215     }
3216 
3217     List<String> commands = new ArrayList<String>();
3218     
3219     addAntCommands(commands);
3220     
3221     System.out.println("\n##################################");
3222     System.out.println("Building client with command:\n" + clientDir.getAbsolutePath() + "> " 
3223         + convertCommandsIntoCommand(commands) + "\n");
3224     
3225     CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
3226         true, true, null, clientDir, null, true);
3227     
3228     if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
3229       System.out.println("stderr: " + commandResult.getErrorText());
3230     }
3231     if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
3232       System.out.println("stdout: " + commandResult.getOutputText());
3233     }
3234 
3235     System.out.println("\nEnd building client");
3236     System.out.println("##################################\n");
3237     
3238   }
3239 
3240   /**
3241    * build client API
3242    * @param duoDir
3243    */
3244   private void buildDuo(File duoDir) {
3245     if (!duoDir.exists() || duoDir.isFile()) {
3246       throw new RuntimeException("Cant find duo: " + duoDir.getAbsolutePath());
3247     }
3248     
3249     File duoBuildToDir = new File(duoDir.getAbsoluteFile() + File.separator + "dist" + File.separator + "bin");
3250     
3251     boolean rebuildDuo = true;
3252     
3253     if (duoBuildToDir.exists()) {
3254       System.out.print("Grouper duo has been built in the past, do you want it rebuilt? (t|f) [t]: ");
3255       rebuildDuo = readFromStdInBoolean(true, "grouperInstaller.autorun.rebuildDuoAfterHavingBeenBuilt");
3256     }
3257     
3258     if (!rebuildDuo) {
3259       return;
3260     }
3261 
3262     List<String> commands = new ArrayList<String>();
3263     
3264     addAntCommands(commands);
3265     
3266     System.out.println("\n##################################");
3267     System.out.println("Building duo with command:\n" + duoDir.getAbsolutePath() + "> " 
3268         + convertCommandsIntoCommand(commands) + "\n");
3269     
3270     CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
3271         true, true, null, duoDir, null, true);
3272     
3273     if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
3274       System.out.println("stderr: " + commandResult.getErrorText());
3275     }
3276     if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
3277       System.out.println("stdout: " + commandResult.getOutputText());
3278     }
3279 
3280     System.out.println("\nEnd building duo");
3281     System.out.println("##################################\n");
3282     
3283   }
3284 
3285   /**
3286    * build box API
3287    * @param boxDir
3288    */
3289   private void buildBox(File boxDir) {
3290     if (!boxDir.exists() || boxDir.isFile()) {
3291       throw new RuntimeException("Cant find box: " + boxDir.getAbsolutePath());
3292     }
3293     
3294     File duoBuildToDir = new File(boxDir.getAbsoluteFile() + File.separator + "dist" + File.separator + "bin");
3295     
3296     boolean rebuildBox = true;
3297     
3298     if (duoBuildToDir.exists()) {
3299       System.out.print("Grouper box has been built in the past, do you want it rebuilt? (t|f) [t]: ");
3300       rebuildBox = readFromStdInBoolean(true, "grouperInstaller.autorun.rebuildBoxAfterHavingBeenBuilt");
3301     }
3302     
3303     if (!rebuildBox) {
3304       return;
3305     }
3306 
3307     List<String> commands = new ArrayList<String>();
3308     
3309     addAntCommands(commands);
3310     
3311     System.out.println("\n##################################");
3312     System.out.println("Building box with command:\n" + boxDir.getAbsolutePath() + "> " 
3313         + convertCommandsIntoCommand(commands) + "\n");
3314     
3315     CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
3316         true, true, null, boxDir, null, true);
3317     
3318     if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
3319       System.out.println("stderr: " + commandResult.getErrorText());
3320     }
3321     if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
3322       System.out.println("stdout: " + commandResult.getOutputText());
3323     }
3324 
3325     System.out.println("\nEnd building box");
3326     System.out.println("##################################\n");
3327     
3328   }
3329 
3330   /**
3331    * admin
3332    */
3333   private void mainAdminLogic() {
3334     
3335     this.version = GrouperInstallerUtils.propertiesValue("grouper.version", true);
3336 
3337     GrouperInstallerAdminAction grouperInstallerAdminAction = 
3338         (GrouperInstallerAdminAction)promptForEnum(
3339             "What admin action do you want to do (manage, upgradeTask, develop)? : ",
3340             "grouperInstaller.autorun.adminAction", GrouperInstallerAdminAction.class);
3341     
3342     switch(grouperInstallerAdminAction) {
3343       case manage:
3344         mainManageLogic();
3345         break;
3346 
3347       case develop:
3348         mainDevelopLogic();
3349         break;
3350 
3351       case upgradeTask:
3352         mainUpgradeTaskLogic();
3353         break;
3354     }
3355         
3356   }
3357 
3358   /**
3359    * admin manage
3360    */
3361   private void mainManageLogic() {
3362     
3363     //####################################
3364     //Find out what directory to install to.  This ends in a file separator
3365     this.grouperInstallDirectoryString = grouperInstallDirectory();
3366 
3367     GrouperInstallerManageAction grouperInstallerManageAction = null;
3368 
3369     while (true) {
3370       grouperInstallerManageAction = 
3371           (GrouperInstallerManageAction)promptForEnum(
3372               "What do you want to manage (logs, services, back, exit)? : ",
3373               "grouperInstaller.autorun.manageAction", GrouperInstallerManageAction.class);
3374 
3375       switch(grouperInstallerManageAction) {
3376         case logs:
3377       
3378           adminManageLogs();
3379   
3380           break;
3381         case services:
3382 
3383           adminManageServices();
3384           
3385           break;
3386         case exit:
3387           
3388           System.exit(0);
3389           
3390           break;
3391         case back:
3392           
3393           this.mainAdminLogic();
3394   
3395           break;
3396       }
3397       
3398       System.out.print("Press <enter> to continue or type 'exit' to end: ");
3399       String result = readFromStdIn("grouperInstaller.autorun.manageContinue");
3400       if (GrouperInstallerUtils.equalsIgnoreCase(result, "exit")) {
3401         System.exit(0);
3402       }
3403       //add some space
3404       System.out.println("");
3405     }
3406   }
3407 
3408   /**
3409    * admin manage
3410    */
3411   private void mainDevelopLogic() {
3412     
3413     GrouperInstallerDevelopAction grouperInstallerDevelopAction = null;
3414 
3415     while (true) {
3416       grouperInstallerDevelopAction = 
3417           (GrouperInstallerDevelopAction)promptForEnum(
3418               "What do you want to develop (translate, back, exit)? : ",
3419               "grouperInstaller.autorun.developAction", GrouperInstallerDevelopAction.class);
3420 
3421       switch(grouperInstallerDevelopAction) {
3422         case translate:
3423 
3424           adminTranslate();
3425 
3426           break;
3427         case exit:
3428 
3429           System.exit(0);
3430 
3431           break;
3432         case back:
3433           
3434           this.mainAdminLogic();
3435   
3436           break;
3437       }
3438       
3439       System.out.print("Press <enter> to continue or type 'exit' to end: ");
3440       String result = readFromStdIn("grouperInstaller.autorun.developContinue");
3441       if (GrouperInstallerUtils.equalsIgnoreCase(result, "exit")) {
3442         System.exit(0);
3443       }
3444       //add some space
3445       System.out.println("");
3446     }
3447   }
3448 
3449   /**
3450    * try 10 times to get enum
3451    * @param prompt
3452    * @param configKey
3453    * @param theClass
3454    * @return the object
3455    */
3456   public static Object promptForEnum(String prompt, String configKey, Class<?> theClass) {
3457     return promptForEnum(prompt, configKey, theClass, null, null);
3458   }
3459 
3460   /**
3461    * try 10 times to get enum
3462    * @param prompt
3463    * @param configKey
3464    * @param enumClass
3465    * @param theDefault
3466    * @param configKeyForDefault
3467    * @return the object
3468    */
3469   public static Object promptForEnum(String prompt, String configKey, Class<?> enumClass, Object theDefault, String configKeyForDefault) {
3470 
3471     //if we are using a config key
3472     if (!GrouperInstallerUtils.isBlank(configKeyForDefault)) {
3473       String defaultAction = GrouperInstallerUtils.propertiesValue(configKeyForDefault, false);
3474       if (!GrouperInstallerUtils.isBlank(defaultAction)) {
3475         theDefault = GrouperInstallerUtils.callMethod(enumClass, null, "valueOfIgnoreCase",
3476             new Class<?>[]{String.class, boolean.class, boolean.class}, new Object[]{defaultAction, true, true});
3477       }
3478       defaultAction = GrouperInstallerUtils.defaultIfBlank(defaultAction, "install");
3479     }
3480     if (theDefault != null) {
3481       prompt += "[" + ((Enum<?>)theDefault).name() + "]: ";
3482     }
3483     
3484     for (int i=0;i<10;i++) {
3485       System.out.print(prompt);
3486       String input = readFromStdIn(configKey);
3487       if (GrouperInstallerUtils.isBlank(input)) {
3488         if (theDefault != null) {
3489           return theDefault;
3490         }
3491         System.out.println("Input is required");
3492         continue;
3493       }
3494 
3495       //call a static method via reflection
3496       Object result = GrouperInstallerUtils.callMethod(enumClass, null, "valueOfIgnoreCase",
3497           new Class<?>[]{String.class, boolean.class, boolean.class}, new Object[]{input, false, false});
3498       if (result != null) {
3499         return result;
3500       } 
3501     }
3502     throw new RuntimeException("Cant find valid answer!!!!");
3503   }
3504   
3505   /**
3506    * 
3507    */
3508   private void adminManageServices() {
3509     
3510     //see what we are upgrading: api, ui, ws, client
3511     GrouperInstallerAdminManageService grouperInstallerAdminManageService = 
3512         (GrouperInstallerAdminManageService)promptForEnum(
3513             "What service do you want to manage?  database, tomcat, grouperDaemon? : ",
3514             "grouperInstaller.autorun.serviceToManage", GrouperInstallerAdminManageService.class);
3515 
3516     GrouperInstallerAdminManageServiceAction grouperInstallerAdminManageServiceAction = 
3517         (GrouperInstallerAdminManageServiceAction)promptForEnum(
3518             "What " + grouperInstallerAdminManageService + " action do you want to perform?  stop, start, restart, status? : ",
3519             "grouperInstaller.autorun.serviceToManageAction", GrouperInstallerAdminManageServiceAction.class);
3520 
3521     switch (grouperInstallerAdminManageService) {
3522       case grouperDaemon:
3523         adminManageGrouperDaemon(grouperInstallerAdminManageServiceAction);
3524         
3525         break;
3526       case database:
3527 
3528         adminManageDatabase(grouperInstallerAdminManageServiceAction);
3529         break;
3530        case tomcat:
3531 
3532         adminManageTomcat(grouperInstallerAdminManageServiceAction);
3533         
3534 
3535         break;
3536 
3537     }
3538     
3539   }
3540 
3541   /**
3542    * translate a ui text file
3543    */
3544   private void adminTranslate() {
3545 
3546     System.out.println("What is the location of the grouper.text.en.us.base.properties file: ");
3547     String grouperTextEnUsBasePropertiesName = readFromStdIn("grouperInstaller.autorun.translate.from");
3548 
3549     if (GrouperInstallerUtils.isBlank(grouperTextEnUsBasePropertiesName)) {
3550       System.out.println("The location of the grouper.text.en.us.base.properties file is required!");
3551       System.exit(1);
3552     }
3553 
3554     File grouperTextEnUsBasePropertiesFile = new File(grouperTextEnUsBasePropertiesName);
3555 
3556     if (grouperTextEnUsBasePropertiesFile.isDirectory()) {
3557       grouperTextEnUsBasePropertiesName = GrouperInstallerUtils.stripLastSlashIfExists(grouperTextEnUsBasePropertiesName);
3558       grouperTextEnUsBasePropertiesName = grouperTextEnUsBasePropertiesName + File.separator + "grouper.text.en.us.base.properties";
3559       grouperTextEnUsBasePropertiesFile = new File(grouperTextEnUsBasePropertiesName);
3560     }
3561 
3562     if (!grouperTextEnUsBasePropertiesFile.isFile() || !grouperTextEnUsBasePropertiesFile.exists()) {
3563       System.out.println("The grouper.text.en.us.base.properties file is not found! " + grouperTextEnUsBasePropertiesFile.getAbsolutePath());
3564       System.exit(1);
3565     }
3566     
3567     System.out.println("What is the location of the translated file: ");
3568     String grouperTranslatedBasePropertiesName = readFromStdIn("grouperInstaller.autorun.translate.to");
3569 
3570     if (GrouperInstallerUtils.isBlank(grouperTextEnUsBasePropertiesName)) {
3571       System.out.println("The location of the translated file is required!");
3572       System.exit(0);
3573     }
3574 
3575     File grouperTranslatedBasePropertiesFile = new File(grouperTranslatedBasePropertiesName);
3576 
3577     if (!grouperTranslatedBasePropertiesFile.isFile() || !grouperTranslatedBasePropertiesFile.exists()) {
3578       System.out.println("The translated file is not found! " + grouperTextEnUsBasePropertiesFile.getAbsolutePath());
3579       System.exit(0);
3580     }
3581     
3582     //backup the existing file
3583     File grouperTranslatedBasePropertiesFileBak = new File(GrouperInstallerUtils.prefixOrSuffix(
3584         grouperTranslatedBasePropertiesFile.getAbsolutePath(), ".properties", true) + "." 
3585         + new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss_SSS").format(new Date()) + ".properties");
3586     
3587     // GrouperInstallerUtils.newlineFromFile(GrouperInstallerUtils.readFileIntoString(grouperTranslatedBasePropertiesFile));
3588     String newline = "\n";
3589     
3590     GrouperInstallerUtils.copyFile(grouperTranslatedBasePropertiesFile, grouperTranslatedBasePropertiesFileBak);
3591     System.out.println("The translated file was backed up to: " + grouperTranslatedBasePropertiesFileBak.getAbsolutePath());
3592     
3593     System.out.print("Do you want to edit this file inline (if not will just run a report) (t|f) [t]: ");
3594     boolean editInline = readFromStdInBoolean(true, "grouperInstaller.translate.editInline");
3595    
3596     StringBuilder output = new StringBuilder();
3597     
3598     String grouperTextEnUsBasePropertiesContents = GrouperInstallerUtils.readFileIntoString(grouperTextEnUsBasePropertiesFile);
3599     String grouperTranslatedBasePropertiesContents = GrouperInstallerUtils.readFileIntoString(grouperTranslatedBasePropertiesFile);
3600 
3601     //go through the original properties line by line
3602     String[] grouperTextEnUsBasePropertiesLines = GrouperInstallerUtils.splitLines(grouperTextEnUsBasePropertiesContents);
3603     String[] grouperTranslatedBasePropertiesLines = GrouperInstallerUtils.splitLines(grouperTranslatedBasePropertiesContents);
3604     Properties existingTranslatedLinesByKey = new Properties();
3605 
3606     //make raw properties
3607     for (String grouperTranslatedBasePropertiesLine : grouperTranslatedBasePropertiesLines) {
3608       int equalsIndex = grouperTranslatedBasePropertiesLine.indexOf('=');
3609       if (equalsIndex != -1) {
3610         String propertyName = GrouperInstallerUtils.prefixOrSuffix(grouperTranslatedBasePropertiesLine, "=", true).trim();
3611         String propertyValue = GrouperInstallerUtils.prefixOrSuffix(grouperTranslatedBasePropertiesLine, "=", false).trim();
3612         if (!GrouperInstallerUtils.isBlank(propertyValue)) {
3613           existingTranslatedLinesByKey.put(propertyName, grouperTranslatedBasePropertiesLine);
3614         }
3615       }
3616     }
3617 
3618     StringBuilder propertyAndComments = new StringBuilder();
3619     int diffCount = 0;
3620 
3621     int lineCount = 1;
3622     
3623     for (String grouperTextEnUsBasePropertiesLine: grouperTextEnUsBasePropertiesLines) {
3624       
3625       Map<String, Object> debugMap = new LinkedHashMap<String, Object>();
3626       
3627       grouperTextEnUsBasePropertiesLine = grouperTextEnUsBasePropertiesLine.trim();
3628       
3629       boolean isBlank = GrouperInstallerUtils.isBlank(grouperTextEnUsBasePropertiesLine);
3630       boolean isComment = grouperTextEnUsBasePropertiesLine.trim().startsWith("#");
3631       boolean isProperty = !isBlank && !isComment && grouperTextEnUsBasePropertiesLine.contains("=");
3632       
3633       if (!isBlank && !isComment && !isProperty) {
3634         System.out.print("Line " + lineCount + " is not a blank, comment, or property, hit <enter> to continue");
3635         readFromStdIn("grouperInstaller.autorun.translateIssueContinue");
3636       }
3637       
3638       debugMap.put("isBlank", isBlank);
3639       debugMap.put("isComment", isComment);
3640       debugMap.put("isProperty", isProperty);
3641       
3642       propertyAndComments.append(newline).append(grouperTextEnUsBasePropertiesLine);
3643 
3644       if (!isProperty) {
3645         output.append(grouperTextEnUsBasePropertiesLine).append(newline);
3646         debugMap.put("clearPropertyAndComments", false);
3647       } else {
3648         int equalsIndex = grouperTextEnUsBasePropertiesLine.indexOf('=');
3649         if (equalsIndex == -1) {
3650           //shouldnt happen
3651           throw new RuntimeException("Coding error: " + grouperTextEnUsBasePropertiesLine);
3652         }
3653         
3654         String propertyName = grouperTextEnUsBasePropertiesLine.substring(0, equalsIndex).trim();
3655 
3656         debugMap.put("propertyName", propertyName);
3657 
3658         String translatedPropertyLine = existingTranslatedLinesByKey.getProperty(propertyName);
3659         
3660         debugMap.put("hasTranslation", !GrouperInstallerUtils.isBlank(translatedPropertyLine));
3661 
3662         // see if there is already a translation
3663         if (!GrouperInstallerUtils.isBlank(translatedPropertyLine)) {
3664  
3665           //just append everything to the new file
3666           output.append(translatedPropertyLine).append(newline);
3667           
3668         } else {
3669           diffCount++;
3670 
3671           //there is no translation
3672           if (!editInline) {
3673             System.out.println(diffCount + ": Translate line " + lineCount + ":");
3674           }
3675 
3676           System.out.println("");
3677           System.out.println(propertyAndComments.toString().trim() + newline);
3678           
3679           //there is no translation
3680           if (editInline) {
3681             System.out.print("\n" + diffCount + ": Enter a translation for line " + lineCount + ":");
3682             String translatedValue = readFromStdIn("autorun.translate.value");
3683             
3684             output.append(propertyName).append("=").append(translatedValue).append(newline);
3685 
3686           } else {
3687             
3688             output.append(propertyName).append("=").append(newline);
3689             
3690           }
3691           
3692         }
3693         debugMap.put("clearPropertyAndComments", true);
3694         propertyAndComments = new StringBuilder();
3695         
3696       }
3697       
3698       if (GrouperInstallerUtils.propertiesValueBoolean("printDebugInfo", false, false)) {
3699         System.out.println(GrouperInstallerUtils.mapToString(debugMap));
3700       }
3701       
3702       lineCount++;
3703     }
3704     GrouperInstallerUtils.saveStringIntoFile(grouperTranslatedBasePropertiesFile, output.toString(), true, true);
3705     
3706     if (diffCount == 0) {
3707       System.out.println("The translated file is complete");
3708     } else {
3709       if (!editInline) {
3710         System.out.println("You have " + diffCount + " missing properties, they need translation.");
3711       } else {
3712         System.out.println("You translated " + diffCount + " missing properties.");
3713       }
3714     }
3715     System.exit(0);
3716   }
3717 
3718   /**
3719    * 
3720    * @param grouperInstallerAdminManageServiceAction
3721    */
3722   private void adminManageTomcat(
3723       GrouperInstallerAdminManageServiceAction grouperInstallerAdminManageServiceAction) {
3724     //tomcat dir
3725     File catalinaServerXmlFile = new File(this.grouperInstallDirectoryString + "conf" + File.separator + "server.xml");
3726     if (!catalinaServerXmlFile.exists()) {
3727       //if used the webapps dir
3728       catalinaServerXmlFile = new File(this.grouperInstallDirectoryString + ".." + File.separator + ".." + File.separator + "conf" + File.separator + "server.xml");
3729     }
3730     //normal installer dir
3731     if (!catalinaServerXmlFile.exists()) {
3732       catalinaServerXmlFile = new File(this.grouperInstallDirectoryString + File.separator 
3733           + "apache-tomcat-" + this.tomcatVersion() + "" + File.separator + "conf" + File.separator + "server.xml");
3734     }
3735 
3736     this.untarredTomcatDir = catalinaServerXmlFile.getParentFile().getParentFile();       
3737 
3738     //  /Server/Service/Connector <Connector port="8080" protocol="HTTP/1.1" 
3739     this.tomcatHttpPort = GrouperInstallerUtils.xpathEvaluateAttributeInt(catalinaServerXmlFile, "/Server/Service/Connector[@protocol='HTTP/1.1']", "port", -1);
3740 
3741     System.out.print("Enter the default IP address for checking ports (just hit enter to accept the default unless on a machine with no network, might want to change to 127.0.0.1): [0.0.0.0]: ");
3742     this.defaultIpAddress = readFromStdIn("grouperInstaller.autorun.defaultIpAddressForPorts");
3743     
3744     if (GrouperInstallerUtils.isBlank(this.defaultIpAddress)) {
3745       this.defaultIpAddress = "0.0.0.0";
3746     }
3747 
3748     switch (grouperInstallerAdminManageServiceAction) {
3749       case stop:
3750       case start:
3751       case restart:
3752         
3753         tomcatBounce(grouperInstallerAdminManageServiceAction.name().toString());
3754         break;
3755       case status:
3756         
3757         if (!GrouperInstallerUtils.portAvailable(this.tomcatHttpPort, this.defaultIpAddress)) {
3758           System.out.println("Tomcat is running.  It is detected to be listening on port: " + this.tomcatHttpPort);
3759         } else {
3760           System.out.println("Tomcat is stopped.  It is not detected to be listening on port: " + this.tomcatHttpPort);
3761         }
3762         break;
3763     }
3764   }
3765 
3766   /**
3767    * 
3768    * @param grouperInstallerAdminManageServiceAction
3769    */
3770   private void adminManageDatabase(
3771       GrouperInstallerAdminManageServiceAction grouperInstallerAdminManageServiceAction) {
3772     List<File> grouperHibernatePropertiesFiles = GrouperInstallerUtils.fileListRecursive(new File(this.grouperInstallDirectoryString), "grouper.hibernate.properties");
3773     
3774     if (GrouperInstallerUtils.length(grouperHibernatePropertiesFiles) == 0) {
3775       System.out.println("Cant find a grouper.hibernate.properties in the install directory: " + this.grouperInstallDirectoryString);
3776     }
3777 
3778     //lets see which one
3779     File grouperHibernatePropertiesFileLocal = null;
3780     String url = null;
3781     
3782     for (File file : grouperHibernatePropertiesFiles) {
3783       Properties grouperHibernateProperties = GrouperInstallerUtils.propertiesFromFile(file);
3784       String urlFromFile = grouperHibernateProperties.getProperty("hibernate.connection.url");
3785 
3786       if (url == null) {
3787         grouperHibernatePropertiesFileLocal = file;
3788         url = urlFromFile;
3789       }
3790       if (!GrouperInstallerUtils.equals(url, urlFromFile)) {
3791         System.out.println("You have " + grouperHibernatePropertiesFiles.size() 
3792           + " grouper.hibernate.properties files in the install directory "
3793           + this.grouperInstallDirectoryString + " with different urls: " + url + ", " + urlFromFile
3794           + ", sync up your config files or specify an install directory that has one grouper.hibernate.properties"); 
3795         for (File current : grouperHibernatePropertiesFiles) {
3796           System.out.println("\n  " + current.getAbsolutePath());
3797         }
3798       }
3799     }
3800     
3801     Properties grouperHibernateProperties = GrouperInstallerUtils.propertiesFromFile(grouperHibernatePropertiesFileLocal);
3802 
3803     this.dbUrl = url;
3804     this.dbUser = GrouperInstallerUtils.defaultString(grouperHibernateProperties.getProperty("hibernate.connection.username"));
3805     this.dbPass = GrouperInstallerUtils.defaultString(grouperHibernateProperties.getProperty("hibernate.connection.password"));
3806     this.giDbUtils = new GiDbUtils(this.dbUrl, this.dbUser, this.dbPass);
3807     this.giDbUtils.registerDriverOnce(this.grouperInstallDirectoryString);
3808     
3809     System.out.println("grouper.hibernate.properties read from: " + grouperHibernatePropertiesFileLocal.getAbsolutePath());
3810     System.out.println("Database URL (hibernate.connection.url from grouper.hibernate.properties) is: " + this.dbUrl);
3811     
3812     if (grouperInstallerAdminManageServiceAction == GrouperInstallerAdminManageServiceAction.status) {
3813       System.out.println("Trying query: " + this.giDbUtils.checkConnectionQuery());
3814       //check connection to database
3815       Exception exception = this.giDbUtils.checkConnection();
3816       if (exception == null) {
3817         System.out.println("Database is up and connection from Java successful.");
3818       } else {
3819         System.out.print("Database could not be connected to from Java.  Perhaps it is down or there is a network problem?\n"
3820             + "  Do you want to see the stacktrace from the connection error? (t|f) [f]: ");
3821         boolean showStack = readFromStdInBoolean(false, "grouperInstaller.autorun.printStackFromDbConnectionError");
3822         if (showStack) {
3823           exception.printStackTrace();
3824         }
3825       }
3826     } else {          
3827       
3828       System.out.println("Error: you are using an external database, (URL above), you need to " + grouperInstallerAdminManageServiceAction + " that database yourself");
3829         
3830     }
3831   }
3832 
3833   /**
3834    * 
3835    * @param grouperInstallerAdminManageServiceAction
3836    */
3837   private void adminManageGrouperDaemon(
3838       GrouperInstallerAdminManageServiceAction grouperInstallerAdminManageServiceAction) {
3839     boolean done = false;
3840     if (!GrouperInstallerUtils.isWindows()) {
3841       
3842       System.out.println("In unix you should have a /etc/init.d or launchctl script which manages the grouper daemon (see details on wiki).");
3843       System.out.print("If you have a service configured please enter name or <enter> to continue without a service: ");
3844       String daemonName = readFromStdIn("grouperInstaller.autorun.grouperDaemonNameOrContinue");
3845       if (!GrouperInstallerUtils.isBlank(daemonName)) {
3846         done = true;
3847         boolean isService = true;
3848         String command = "/sbin/service";
3849         if (!new File(command).exists()) {
3850           command = "/usr/sbin/service";
3851         }
3852         if (!new File(command).exists()) {
3853           command = "/bin/launchctl";
3854           isService = false;
3855         }
3856         if (!new File(command).exists()) {
3857           System.out.println("Cannot find servie command, looked for /sbin/service, /usr/sbin/service, and /bin/launchctl.  "
3858               + "Your version of unix services is not supported.  Contact the Grouper support team.");
3859           System.exit(1);
3860         }
3861         if (isService) {
3862           List<String> commands = new ArrayList<String>();
3863           commands.add(command);
3864           commands.add(daemonName);
3865           commands.add(grouperInstallerAdminManageServiceAction.name());
3866           
3867           System.out.println(grouperInstallerAdminManageServiceAction + " " + daemonName
3868               + " with command: " + convertCommandsIntoCommand(commands) + "\n");
3869 
3870           GrouperInstallerUtils.execCommand(
3871               GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
3872               new File(this.grouperInstallDirectoryString), null, false, false, true);
3873         } else {
3874           // <pid> <status> mytask
3875           if (grouperInstallerAdminManageServiceAction == GrouperInstallerAdminManageServiceAction.status) {
3876             // launchctl list | grep mytask
3877             List<String> commandsToRun = new ArrayList<String>();
3878             commandsToRun.add(shCommand());
3879             commandsToRun.add("-c");
3880             commandsToRun.add(command + " list | " + grepCommand() + " " + daemonName);
3881             
3882             System.out.println(grouperInstallerAdminManageServiceAction + " " + daemonName
3883                 + " with command: " + convertCommandsIntoCommand(commandsToRun) + "\n");
3884 
3885             GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commandsToRun, String.class), true, true, null, 
3886                 new File(this.grouperInstallDirectoryString), null, false, false, true);
3887             
3888           } else {
3889             // launchctl start|stop mytask
3890             if (grouperInstallerAdminManageServiceAction == GrouperInstallerAdminManageServiceAction.stop 
3891                 || grouperInstallerAdminManageServiceAction == GrouperInstallerAdminManageServiceAction.restart) {
3892               //stop daemon
3893               List<String> commands = new ArrayList<String>();
3894               commands.add(command);
3895               commands.add("stop");
3896               commands.add(daemonName);
3897               
3898               System.out.println("stopping " + daemonName
3899                   + " with command: " + convertCommandsIntoCommand(commands) + "\n");
3900 
3901               GrouperInstallerUtils.execCommand(
3902                   GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
3903                   new File(this.grouperInstallDirectoryString), null, false, false, true);
3904               
3905             }
3906  
3907             if (grouperInstallerAdminManageServiceAction == GrouperInstallerAdminManageServiceAction.restart) {
3908               GrouperInstallerUtils.sleep(3000);
3909             }
3910  
3911             if (grouperInstallerAdminManageServiceAction == GrouperInstallerAdminManageServiceAction.start 
3912                 || grouperInstallerAdminManageServiceAction == GrouperInstallerAdminManageServiceAction.restart) {
3913               //start daemon
3914               List<String> commands = new ArrayList<String>();
3915               commands.add(command);
3916               commands.add("start");
3917               commands.add(daemonName);
3918               
3919               System.out.println("starting " + daemonName
3920                   + " with command: " + convertCommandsIntoCommand(commands) + "\n");
3921 
3922               GrouperInstallerUtils.execCommand(
3923                   GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
3924                   new File(this.grouperInstallDirectoryString), null, false, false, true);
3925               
3926               GrouperInstallerUtils.sleep(5000);
3927             }
3928           }              
3929         }
3930       }
3931     }
3932 
3933     if (!done) {
3934       
3935       if (new File(this.grouperInstallDirectoryString + "grouper.apiBinary-" + this.version).exists()) {
3936         this.untarredApiDir = new File(this.grouperInstallDirectoryString + "grouper.apiBinary-" + this.version);
3937       }
3938       
3939       //this is for loader dir
3940       if (new File(this.grouperInstallDirectoryString + "WEB-INF").exists()) {
3941         this.upgradeExistingApplicationDirectoryString = this.grouperInstallDirectoryString;
3942       } else if (new File(this.grouperInstallDirectoryString + "bin").exists()) {
3943         this.upgradeExistingApplicationDirectoryString = this.grouperInstallDirectoryString;
3944       }
3945       
3946       String gshCommandLocal = gshCommand();
3947       if (gshCommandLocal.endsWith(".sh")) {
3948         gshCommandLocal = gshCommandLocal.substring(0, gshCommandLocal.length()-".sh".length());
3949       }
3950       if (gshCommandLocal.endsWith(".bat")) {
3951         gshCommandLocal = gshCommandLocal.substring(0, gshCommandLocal.length()-".bat".length());
3952       }
3953       
3954       if (!GrouperInstallerUtils.isWindows()) {
3955         if (gshCommandLocal.contains(" ")) {
3956           System.out.println("On unix the gsh command cannot contain whitespace!");
3957           System.exit(1);
3958         }
3959       }
3960       
3961       // ps -ef | grep -- -loader | grep -v grep
3962       List<String> psCommands = new ArrayList<String>();
3963       psCommands.add(shCommand());
3964       psCommands.add("-c");
3965       psCommands.add( psCommand() + " -ef | " + grepCommand() + " " + gshCommandLocal + " | " 
3966           + grepCommand() + " -- -loader | " + grepCommand() + " -v grep");
3967 
3968       //unix
3969       //appadmin 14477     1  0 01:24 pts/0    00:00:00 /bin/sh /opt/tomcats/tomcat_d_gsh/webapps/grouper_v2_2/WEB-INF/bin/gsh -loader
3970       //appadmin 14478 14477 92 01:24 pts/0    00:00:03 /opt/java6/bin/java -Xms64m -Xmx750m -Dgrouper.home=/opt/tomcats/tomcat_d_gsh/webapps/grouper_v2_2/WEB-INF/bin/../ -classpath /opt/tomcats/tomcat_d_gsh/webapps/grouper_v2_2/WEB-INF/bin/../classes:/opt/tomcats/tomcat_d_gsh/webapps/grouper_v2_2/WEB-INF/bin/../lib/*: edu.internet2.middleware.grouper.app.gsh.GrouperShellWrapper -loader
3971       
3972       //mac
3973       //0     1     0   0 Sun06PM ??         1:15.38 /sbin/launchd
3974       //0    45     1   0 Sun06PM ??         0:06.80 /usr/sbin/syslogd
3975       
3976       Pattern pidPattern = Pattern.compile("^[^\\s]+\\s+([^\\s]+)\\s+.*$");
3977       
3978       if (grouperInstallerAdminManageServiceAction == GrouperInstallerAdminManageServiceAction.stop 
3979           || grouperInstallerAdminManageServiceAction == GrouperInstallerAdminManageServiceAction.restart) {
3980 
3981         if (GrouperInstallerUtils.isWindows()) {
3982           System.out.print("In windows you need to find the java process in task manager and kill it, press <enter> to continue... ");
3983           readFromStdIn("grouperInstaller.autorun.enterToContinueWindowsCantKillProcess");
3984         } else {
3985 
3986           System.out.println("Stopping the grouper daemon is not an exact science, be careful!");
3987           System.out.println("This script will find the process id of the daemon and kill it.  Make it is correct!");
3988           System.out.println("Finding the grouper daemon process with: " + convertCommandsIntoCommand(psCommands));
3989 
3990           //stop daemon
3991           CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(psCommands, String.class), false, true, null, 
3992              new File(this.grouperInstallDirectoryString), null, false, false, true);
3993           if (commandResult.getExitCode() != 0 && !GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
3994             throw new RuntimeException("Could not execute: " + convertCommandsIntoCommand(psCommands) 
3995                + "\n" + commandResult.getErrorText()
3996                + "\n" + commandResult.getOutputText());
3997           }
3998           String outputText = GrouperInstallerUtils.defaultString(commandResult.getOutputText());
3999           if (GrouperInstallerUtils.isBlank(outputText)) {
4000             System.out.println("Cannot find the grouper daemon process, it is not running");
4001           } else {
4002             outputText = outputText.replace('\r', '\n');
4003             outputText = GrouperInstallerUtils.replace(outputText, "\r", "\n");
4004             List<String> lines = GrouperInstallerUtils.splitTrimToList(outputText, "\n");
4005             int MAX_LINES = 2;
4006             if (lines.size() > MAX_LINES) {
4007               System.out.println("Found more output than expected, please examine the services on your system and kill them manually");
4008               for (String line : lines) {
4009                 System.out.println(line);
4010               }
4011             } else {
4012               Set<String> pidsDone = new HashSet<String>();
4013               for (int i=0; i<MAX_LINES; i++) {
4014                 // ^[^\s]+\s+([^\s]+)\s+.*$
4015                 // start, then first thing, then spaces, then second thing is the pic, then spaces, then whatever and end string
4016                 Matcher matcher = pidPattern.matcher(lines.get(0));
4017                 if (matcher.matches()) {
4018                   String pid = matcher.group(1);
4019                   if (pidsDone.contains(pid)) {
4020                     System.out.println("Could not kill pid " + pid);
4021                     System.exit(1);
4022                   }
4023                   List<String> killCommandList = GrouperInstallerUtils.splitTrimToList(killCommand() + " -KILL " + pid, " ");
4024                   System.out.println("The command to kill the daemon is: " + convertCommandsIntoCommand(killCommandList));
4025                   System.out.print("Found pid " + pid + ", do you want this script to kill it? (t|f) [t]: ");
4026                   boolean killDaemon = readFromStdInBoolean(true, "grouperInstaller.autorun.killPidOfDaemon");
4027                   
4028                   if (killDaemon) {
4029 
4030                     //keep track that we tried this one
4031                     pidsDone.add(pid);
4032                     
4033                     commandResult = GrouperInstallerUtils.execCommand(
4034                         GrouperInstallerUtils.toArray(killCommandList, String.class), true, true, null, 
4035                        null, null, true, false, true);
4036                     
4037                     GrouperInstallerUtils.sleep(5000);
4038 
4039                     //get next line, hopefully first one isnt there anymore, maybe not second either...
4040                     commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(psCommands, String.class), false, true, null, 
4041                        null, null, false, false, true);
4042 
4043                     if (commandResult.getExitCode() != 0 && !GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
4044                       throw new RuntimeException("Could not execute: " + convertCommandsIntoCommand(psCommands) 
4045                          + "\n" + commandResult.getErrorText()
4046                          + "\n" + commandResult.getOutputText());
4047                     }
4048                     
4049                     outputText = GrouperInstallerUtils.defaultString(commandResult.getOutputText());
4050                     if (GrouperInstallerUtils.isBlank(outputText)) {
4051                       break;
4052                     }
4053                     
4054                     outputText = outputText.replace('\r', '\n');
4055                     outputText = GrouperInstallerUtils.replace(outputText, "\r", "\n");
4056                     lines = GrouperInstallerUtils.splitTrimToList(outputText, "\n");
4057                     
4058                   } else {
4059                     break;
4060                   }
4061                 }
4062               }
4063             }
4064           }
4065         }
4066       }
4067 
4068       if (grouperInstallerAdminManageServiceAction == GrouperInstallerAdminManageServiceAction.restart) {
4069         GrouperInstallerUtils.sleep(3000);
4070       }
4071 
4072       if (grouperInstallerAdminManageServiceAction == GrouperInstallerAdminManageServiceAction.start 
4073           || grouperInstallerAdminManageServiceAction == GrouperInstallerAdminManageServiceAction.restart) {
4074         //start
4075         startLoader(false);
4076         GrouperInstallerUtils.sleep(5000);
4077       }
4078 
4079       if (grouperInstallerAdminManageServiceAction == GrouperInstallerAdminManageServiceAction.status && GrouperInstallerUtils.isWindows()) {
4080         System.out.println("Cant get status of loader when running on Windows.  Look in your task manager for a java process (difficult to tell which one).");
4081       } else {
4082                    
4083         //no matter what, status, or other, do a status
4084         if (!GrouperInstallerUtils.isWindows()) {
4085           //stop daemon
4086           System.out.println("Finding the grouper daemon process with: " + convertCommandsIntoCommand(psCommands));
4087           CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(psCommands, String.class), false, true, null, 
4088              null, null, false, false, true);
4089           if (commandResult.getExitCode() != 0 && !GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
4090             throw new RuntimeException("Could not execute: " + convertCommandsIntoCommand(psCommands) 
4091                + "\n" + commandResult.getErrorText()
4092                + "\n" + commandResult.getOutputText());
4093           }
4094           String outputText = GrouperInstallerUtils.defaultString(commandResult.getOutputText());
4095           if (GrouperInstallerUtils.isBlank(outputText)) {
4096             System.out.println("Cannot find the grouper daemon process, it is not running");
4097           } else {
4098             outputText = outputText.replace('\r', '\n');
4099             outputText = GrouperInstallerUtils.replace(outputText, "\r", "\n");
4100             List<String> lines = GrouperInstallerUtils.splitTrimToList(outputText, "\n");
4101             System.out.println("Grouper loader is running, here is the process output:");
4102             for (String line : lines) {
4103               System.out.println(line);
4104             }
4105           }
4106         }
4107       }
4108     }
4109   }
4110 
4111   /**
4112    * admin manage logs
4113    */
4114   private void adminManageLogs() {
4115     //see what we are upgrading: api, ui, ws, client
4116     this.appToUpgrade = grouperAppToUpgradeOrPatch("look at logs for");
4117 
4118     System.out.println("Find where the application is running, then find the log4j.properties in the classpath.");
4119     
4120     switch (this.appToUpgrade) {
4121       case PSP:
4122       case PSPNG:
4123         System.out.println("This runs in the API, so logging for the API will be examined.");
4124         //pass through to API
4125       case API:
4126         System.out.println("The API (generally invoked via GSH) logs to where the log4.properties specifies.");
4127         File log4jPropertiesFile = new File(this.grouperInstallDirectoryString + "grouper.apiBinary-" + this.version + File.separator 
4128             + "conf" + File.separator + "log4j.properties");
4129         
4130         if (!log4jPropertiesFile.exists()) {
4131           
4132           List<File> allFiles = GrouperInstallerUtils.fileListRecursive(new File(this.grouperInstallDirectoryString));
4133           log4jPropertiesFile = null;
4134           boolean multipleFound = false;
4135           for (File file : allFiles) {
4136             if ("log4j.properties".equals(file.getName())) {
4137               if (log4jPropertiesFile != null) {
4138                 multipleFound = true;
4139                 log4jPropertiesFile = null;
4140                 break;
4141               }
4142               log4jPropertiesFile = file;
4143             }
4144           }
4145           if (multipleFound || log4jPropertiesFile == null) {
4146             System.out.print("What is the absolute path of the log4j.properties? : ");
4147             String log4jPropertiesLocation = readFromStdIn("grouperInstaller.autorun.log4jPropertiesLocation");
4148             log4jPropertiesFile = new File(log4jPropertiesLocation);
4149             if (!log4jPropertiesFile.exists()) {
4150               System.out.println("Bad location: " + log4jPropertiesFile.getAbsolutePath());
4151               System.exit(1);
4152             }
4153           }
4154         }
4155         
4156         File logFile = new File(this.grouperInstallDirectoryString  
4157             + "logs" + File.separator + "grouper_error.log");
4158         String grouperHomeWithSlash = this.grouperInstallDirectoryString;
4159 
4160         if (!logFile.exists()) {
4161           logFile = new File(this.grouperInstallDirectoryString + "grouper.apiBinary-" + this.version + File.separator 
4162               + "logs" + File.separator + "grouper_error.log");
4163           grouperHomeWithSlash = this.grouperInstallDirectoryString + "grouper.apiBinary-" + this.version + File.separator;
4164         }              
4165         System.out.println("By default the installer configures the log file to be: " + logFile.getAbsolutePath());
4166         
4167         
4168         analyzeLogFile(log4jPropertiesFile, grouperHomeWithSlash, null, null);
4169         break;
4170       case CLIENT:
4171         System.out.println("The client generally logs to STDOUT.  Check the grouper.client.properties or if there is a log4j.properties in the clients classpath.");
4172         break;
4173       case WS:
4174       case UI:
4175         File catalinaLogFile = new File(this.grouperInstallDirectoryString + "logs");
4176         if (!catalinaLogFile.exists()) {
4177           //if used the webapps dir
4178           catalinaLogFile = new File(this.grouperInstallDirectoryString + ".." + File.separator + ".." + File.separator + "logs");
4179         }
4180         if (!catalinaLogFile.exists()) {
4181           catalinaLogFile = new File(this.grouperInstallDirectoryString + File.separator 
4182               + "apache-tomcat-" + this.tomcatVersion() + "" + File.separator + "logs");
4183         }
4184         
4185         System.out.println("Tomcat logs STDOUT and STDERR to the catalinaErr.log "
4186             + "and catalinaOut.log logfiles, which should be here: " + catalinaLogFile.getAbsolutePath());
4187         if (!catalinaLogFile.exists()) {
4188           System.out.println("Warning: that directory does not exist, so you will need to locate the logs directory for tomcat.");
4189         }
4190         System.out.println("Locate the " + this.appToUpgrade + " application files.");
4191         System.out.println("By default the installer has the " + this.appToUpgrade + " running based on the tomcat server.xml, "
4192             + "but could also run in the webapps dir.");
4193         
4194         File serverXmlFile = new File(catalinaLogFile.getParentFile().getAbsolutePath() + File.separator + "conf" + File.separator + "server.xml");
4195         
4196         if (!serverXmlFile.exists()) {
4197           System.out.println("server.xml not found: " + serverXmlFile.getAbsolutePath());
4198         } else {
4199           //<Context docBase="/Users/mchyzer/tmp/grouperInstaller/grouper.ws-2.3.0/grouper-ws/build/dist/grouper-ws" path="/grouper-ws" reloadable="false"/>
4200           //<Context docBase="/Users/mchyzer/tmp/grouperInstaller/grouper.ui-2.3.0/dist/grouper" path="/grouper" reloadable="false"/>
4201 
4202           System.out.println("The server.xml is located: " + serverXmlFile.getAbsolutePath());
4203 
4204           String tomcatPath = this.appToUpgrade == AppToUpgrade.UI ? "grouper" : "grouper-ws";
4205 
4206           System.out.print("What is the URL starting path? [" + tomcatPath + "]: ");
4207           String newTomcatPath = readFromStdIn(this.appToUpgrade == AppToUpgrade.UI ? "grouperInstaller.autorun.urlPathForUi" : "grouperInstaller.autorun.urlPathForWs");
4208           
4209           if (!GrouperInstallerUtils.isBlank(newTomcatPath)) {
4210             tomcatPath = newTomcatPath;
4211           }
4212 
4213           if (tomcatPath.endsWith("/") || tomcatPath.endsWith("\\")) {
4214             tomcatPath = tomcatPath.substring(0, tomcatPath.length()-1);
4215           }
4216           if (tomcatPath.startsWith("/") || tomcatPath.startsWith("\\")) {
4217             tomcatPath = tomcatPath.substring(1, tomcatPath.length());
4218           }                  
4219           String currentDocBase = GrouperInstallerUtils.xpathEvaluateAttribute(serverXmlFile, 
4220               "Server/Service/Engine/Host/Context[@path='/" + tomcatPath + "']", "docBase");
4221 
4222           if (this.appToUpgrade == AppToUpgrade.UI) {
4223             //<Context docBase="/Users/mchyzer/tmp/grouperInstaller/grouper.ui-2.3.0/dist/grouper" path="/grouper" reloadable="false"/>
4224             System.out.println("Looking for an entry in the server.xml that looks like this:");
4225             System.out.println("  <Context docBase=\""
4226                 + GrouperInstallerUtils.defaultIfBlank(currentDocBase, this.grouperInstallDirectoryString 
4227                     + "grouper.ui-" + this.version + File.separator + "dist" + File.separator 
4228                     + "grouper")
4229                 + "\" path=\"/" + tomcatPath + "\" reloadable=\"false\"/>");
4230 
4231           } else if (this.appToUpgrade == AppToUpgrade.WS) {
4232             //<Context docBase="/Users/mchyzer/tmp/grouperInstaller/grouper.ws-2.3.0/grouper-ws/build/dist/grouper-ws" path="/grouper-ws" reloadable="false"/>
4233             System.out.println("Looking for an entry in the server.xml that looks like this:");
4234             System.out.println("  <Context docBase=\""
4235                 + GrouperInstallerUtils.defaultIfBlank(currentDocBase, this.grouperInstallDirectoryString 
4236                     + "grouper.ws-" + this.version + File.separator + "grouper-ws" 
4237                     + File.separator + "build" + File.separator + "dist" + File.separator 
4238                     + "grouper-ws")
4239                 + "\" path=\"/" + tomcatPath + "\" reloadable=\"false\"/>");
4240           }
4241           
4242           if (!GrouperInstallerUtils.isBlank(currentDocBase)) {
4243             System.out.println("The docBase for the " + tomcatPath + " entry in the server.xml is: " + currentDocBase);
4244           } else {
4245             //check webapps
4246             System.out.println("The docBase could not be found in the server.xml, check in the tomcat" 
4247                 + File.separator + "webapps directory");
4248             currentDocBase = catalinaLogFile.getParentFile().getAbsolutePath() + File.separator + "webapps" + File.separator + tomcatPath;
4249             if (!new File(currentDocBase).exists()) {
4250               System.out.println("Cant find where grouper is linked from tomcat, looked in server.xml and the webapps directory");
4251               currentDocBase = null;
4252             }
4253           }
4254           if (currentDocBase != null) {
4255             log4jPropertiesFile = new File(currentDocBase + File.separator + "WEB-INF" + File.separator + "classes" + File.separator + "log4j.properties");
4256             
4257             analyzeLogFile(log4jPropertiesFile, "${grouper.home}" + File.separator, new File(catalinaLogFile + File.separator + "catalinaOut.log"),
4258                 new File(catalinaLogFile + File.separator + "catalinaErr.log"));
4259           }
4260         }
4261         
4262         break;
4263         
4264       default: 
4265         throw new RuntimeException("Not expecting appToUpgrade: " + this.appToUpgrade + "!");
4266     }
4267     
4268   }
4269   
4270   /**
4271    * 
4272    * @param log4jPropertiesFile
4273    * @param grouperHomeWithSlash
4274    * @param stdoutLocation
4275    * @param stderrLocation
4276    */
4277   private void analyzeLogFile(File log4jPropertiesFile, String grouperHomeWithSlash, File stdoutLocation, File stderrLocation) {
4278     System.out.println("The log4j.properties is located in: " 
4279         + log4jPropertiesFile.getAbsolutePath());
4280 
4281     if (!log4jPropertiesFile.exists()) {
4282     
4283       System.out.println("Error, the log4j.properties file could not be found.");
4284 
4285     } else {
4286       
4287       System.out.println("Examine the log4j.properties to see where it is logging");
4288       
4289       Properties log4jProperties = GrouperInstallerUtils.propertiesFromFile(log4jPropertiesFile);
4290       
4291       //= ERROR, grouper_error")
4292       String rootLoggerValue = log4jProperties.getProperty("log4j.rootLogger");              
4293       
4294       System.out.println("Generally the log4j.rootLogger property shows where logs go, it is set to: " + rootLoggerValue);
4295       
4296       Pattern pattern = Pattern.compile("\\s*[A-Z]+\\s*,\\s*(\\w+)\\s*");
4297       Matcher matcher = pattern.matcher(rootLoggerValue);
4298       if (!matcher.matches()) {
4299         System.out.println("Examine the log4j.properties for more information");
4300       } else {
4301         String logger = matcher.group(1);
4302         System.out.println("The log4j.rootLogger property in log4j.properties is set to: " + logger);
4303         //log4j.appender.grouper_error = org.apache.log4j.DailyRollingFileAppender
4304         //log4j.appender.grouper_error.File = ${grouper.home}logs/grouper_error.log
4305         String logFileName = log4jProperties.getProperty("log4j.appender." + logger + ".File");
4306         if (!GrouperInstallerUtils.isBlank(logFileName)) {
4307           System.out.println("There is a property in log4j.properties: log4j.appender." + logger + ".File = " + logFileName);
4308           if (logFileName.contains("${grouper.home}")) {
4309             logFileName = GrouperInstallerUtils.replace(logFileName, "${grouper.home}", grouperHomeWithSlash);
4310           }
4311           System.out.println("Grouper log should be: " + logFileName);
4312         } else {
4313           //log4j.appender.grouper_stdout = org.apache.log4j.ConsoleAppender
4314           String appender = log4jProperties.getProperty("log4j.appender." + logger);
4315           //log4j.appender.grouper_stderr.Target                    = System.err
4316           String target = log4jProperties.getProperty("log4j.appender." + logger + ".Target");
4317           String targetFriendly = null;
4318           if (GrouperInstallerUtils.equals(target, "System.err")) {
4319             targetFriendly = "STDERR";
4320           } else if (GrouperInstallerUtils.equals(target, "System.out")) {
4321             targetFriendly = "STDOUT";
4322           }
4323           if (GrouperInstallerUtils.equals(appender, "org.apache.log4j.ConsoleAppender") && targetFriendly != null) {
4324             System.out.println("Since log4j.properties log4j.appender." + logger + " = org.apache.log4j.ConsoleAppender you are logging to " + targetFriendly);
4325             if (GrouperInstallerUtils.equals(target, "System.err") && stderrLocation != null) {
4326               System.out.println("Grouper logs should be in " + stderrLocation.getAbsolutePath());
4327             } else if (GrouperInstallerUtils.equals(target, "System.out") && stdoutLocation != null) {
4328               System.out.println("Grouper logs should be in " + stdoutLocation.getAbsolutePath());
4329             }
4330           } else {
4331             System.out.println("Examine the log4j.properties for more information");
4332           }
4333         }
4334       }
4335     }
4336   }
4337   
4338   /**
4339    * admin
4340    */
4341   private void mainUpgradeTaskLogic() {
4342     
4343     GrouperInstallerUpgradeTaskAction grouperInstallerConvertAction = 
4344         (GrouperInstallerUpgradeTaskAction)promptForEnum(
4345             "What upgrade task do you want to do (convertEhcacheXmlToProperties, convertSourcesXmlToProperties, analyzeAndFixJars)? : ",
4346             "grouperInstaller.autorun.upgradeTaskAction", GrouperInstallerUpgradeTaskAction.class);
4347 
4348     switch(grouperInstallerConvertAction) {
4349       case convertEhcacheXmlToProperties:
4350 
4351         System.out.println("Note, you need to convert the ehcache.xml file for each Grouper runtime, e.g. loader, WS, UI.");
4352         System.out.println("Note, you need to be running Grouper 2.3.0 with API patch 35 installed.");
4353         System.out.print("Enter the location of the ehcache.xml file: ");
4354         String convertEhcacheXmlLocation = readFromStdIn("grouperInstaller.autorun.convertEhcacheXmlLocation");
4355 
4356         File ehcacheXmlFile = new File(convertEhcacheXmlLocation);
4357         if (!ehcacheXmlFile.exists()) {
4358           System.out.println("Cant find ehcache.xml: " + ehcacheXmlFile.getAbsolutePath());
4359           System.exit(1);
4360         }
4361 
4362         File grouperCacheBaseProperties = new File(ehcacheXmlFile.getParentFile().getAbsolutePath() + File.separator + "grouper.cache.base.properties");
4363 
4364         {
4365           System.out.print("Enter the location of the grouper.cache.base.properties file [" + grouperCacheBaseProperties.getAbsolutePath() + "]: ");
4366           String grouperCacheBasePropertiesLocation = readFromStdIn("grouperInstaller.autorun.convertEhcacheBasePropertiesLocation");
4367   
4368           if (!GrouperInstallerUtils.isBlank(grouperCacheBasePropertiesLocation)) {
4369             grouperCacheBaseProperties = new File(grouperCacheBasePropertiesLocation);
4370           }
4371         }
4372         
4373         File grouperCacheProperties = new File(ehcacheXmlFile.getParentFile().getAbsolutePath() + File.separator + "grouper.cache.properties");
4374 
4375         {
4376           System.out.print("Enter the location of the grouper.cache.properties file (to be created)  [" + grouperCacheProperties.getAbsolutePath() + "]: ");
4377           String grouperCachePropertiesLocation = readFromStdIn("grouperInstaller.autorun.convertEhcachePropertiesLocation");
4378   
4379           if (!GrouperInstallerUtils.isBlank(grouperCachePropertiesLocation)) {
4380             grouperCacheProperties = new File(grouperCachePropertiesLocation);
4381           }
4382         }
4383         
4384         try {
4385           convertEhcacheXmlToProperties(grouperCacheBaseProperties, grouperCacheProperties,
4386               ehcacheXmlFile.toURI().toURL());
4387         } catch (MalformedURLException mue) {
4388           throw new RuntimeException("Malformed url on " + convertEhcacheXmlLocation);
4389         }
4390 
4391         System.out.println("File was written: " + grouperCacheProperties.getAbsolutePath());
4392 
4393         break;
4394         
4395       case analyzeAndFixJars:
4396         
4397         //Find out what directory to install to.  This ends in a file separator
4398         this.grouperInstallDirectoryString = grouperInstallDirectory();
4399 
4400         reportOnConflictingJars(this.grouperInstallDirectoryString);
4401         
4402         break;
4403         
4404       case convertSourcesXmlToProperties:
4405 
4406         System.out.println("Note, you need to convert the sources.xml file for each Grouper runtime, e.g. loader, WS, UI.");
4407         System.out.println("Note, to use subject sources from subject.properties, you need to be running Grouper 2.3.0+ with API patch 40 installed.");
4408         System.out.print("Enter the location of the sources.xml file: ");
4409         String convertSourcesXmlLocation = readFromStdIn("grouperInstaller.autorun.convertSourceXmlLocation");
4410 
4411         File sourcesXmlFile = new File(convertSourcesXmlLocation);
4412         if (!sourcesXmlFile.exists()) {
4413           System.out.println("Cant find sources.xml: " + sourcesXmlFile.getAbsolutePath());
4414           System.exit(1);
4415         }
4416 
4417         File subjectProperties = new File(sourcesXmlFile.getParentFile().getAbsolutePath() + File.separator + "subject.properties");
4418 
4419         {
4420           System.out.print("Enter the location of the subject.properties file [" + subjectProperties.getAbsolutePath() + "]: ");
4421           String grouperCacheBasePropertiesLocation = readFromStdIn("grouperInstaller.autorun.convertSubjectPropertiesLocation");
4422   
4423           if (!GrouperInstallerUtils.isBlank(grouperCacheBasePropertiesLocation)) {
4424             subjectProperties = new File(grouperCacheBasePropertiesLocation);
4425           }
4426         }
4427         
4428         try {
4429           convertSourcesXmlToProperties(subjectProperties, sourcesXmlFile.toURI().toURL());
4430         } catch (MalformedURLException mue) {
4431           throw new RuntimeException("Malformed url on " + convertSourcesXmlLocation);
4432         }
4433 
4434         System.out.println("File was written: " + subjectProperties.getAbsolutePath());
4435         System.out.println("You should archive your sources.xml and remove it from your project since it is now unused:\n  " 
4436             + sourcesXmlFile.getAbsolutePath());
4437 
4438         break;
4439     }
4440 
4441   }
4442   
4443   /**
4444    * patch grouper
4445    */
4446   private void mainPatchLogic() {
4447     
4448     //####################################
4449     //Find out what directory to upgrade to.  This ends in a file separator
4450     this.grouperTarballDirectoryString = grouperUpgradeTempDirectory();
4451     
4452     //see what we are upgrading: api, ui, ws, client
4453     this.appToUpgrade = grouperAppToUpgradeOrPatch("patch");
4454 
4455     //get the directory where the existing installation is
4456     this.upgradeExistingApplicationDirectoryString = upgradeExistingDirectory();
4457 
4458     GrouperInstallerPatchAction grouperInstallerPatchAction = 
4459         (GrouperInstallerPatchAction)promptForEnum(
4460             "What do you want to do with patches (install, revert, status, fixIndexFile)? ",
4461             "grouperInstaller.autorun.patchAction", GrouperInstallerPatchAction.class, GrouperInstallerPatchAction.install, null);
4462     
4463     switch(grouperInstallerPatchAction) {
4464       case install:
4465         
4466         fixIndexFileIfOk();
4467 
4468         //loop through applications, check patches
4469         this.appToUpgrade.patch(this);
4470 
4471         break;
4472         
4473       case revert:
4474         
4475         fixIndexFileIfOk();
4476 
4477         //look through applications, check for reverts
4478         this.appToUpgrade.revertPatch(this);
4479         break;
4480         
4481       case status:
4482         
4483         fixIndexFileIfOk();
4484 
4485         //print out status for applications
4486         this.appToUpgrade.patchStatus(this);
4487         break;
4488         
4489       case fixIndexFile:
4490         
4491         //print out status for applications
4492         this.appToUpgrade.fixIndexFile(this);
4493         break;
4494         
4495       default:
4496         throw new RuntimeException("Invalid patch action: " + grouperInstallerPatchAction);  
4497     }
4498     
4499   }
4500 
4501   /**
4502    * 
4503    */
4504   public static enum GrouperInstallerPatchAction {
4505 
4506     /** fix index file */
4507     fixIndexFile,
4508     
4509     /** install patches */
4510     install,
4511 
4512     /**
4513      * revert patches
4514      */
4515     revert,
4516     
4517     /**
4518      * get status on patches
4519      */
4520     status;
4521     
4522     /**
4523      * 
4524      * @param string
4525      * @param exceptionIfInvalid
4526      * @param exceptionIfBlank
4527      * @return the action
4528      */
4529     public static GrouperInstallerPatchAction valueOfIgnoreCase(String string, boolean exceptionIfBlank, boolean exceptionIfInvalid) {
4530       return GrouperInstallerUtils.enumValueOfIgnoreCase(GrouperInstallerPatchAction.class, string, exceptionIfBlank, exceptionIfInvalid);
4531     }
4532     
4533   }
4534   
4535   /**
4536    * 
4537    */
4538   public static enum GrouperInstallerAdminAction {
4539 
4540     /** manage */
4541     manage,
4542     
4543     /** develop */
4544     develop,
4545     
4546     /** convert */
4547     upgradeTask;
4548     
4549     /**
4550      * 
4551      * @param string
4552      * @param exceptionIfInvalid
4553      * @param exceptionIfBlank
4554      * @return the action
4555      */
4556     public static GrouperInstallerAdminAction valueOfIgnoreCase(String string, boolean exceptionIfBlank, boolean exceptionIfInvalid) {
4557       return GrouperInstallerUtils.enumValueOfIgnoreCase(GrouperInstallerAdminAction.class, string, exceptionIfBlank, exceptionIfInvalid);
4558     }
4559     
4560   }
4561   
4562   /**
4563    * 
4564    */
4565   public static enum GrouperInstallerUpgradeTaskAction {
4566 
4567     /** analyze and fix jars */
4568     analyzeAndFixJars,
4569     
4570     /** convert */
4571     convertEhcacheXmlToProperties,
4572     
4573     /** convert sources xml */
4574     convertSourcesXmlToProperties;
4575     
4576     /**
4577      * 
4578      * @param string
4579      * @param exceptionIfInvalid
4580      * @param exceptionIfBlank
4581      * @return the action
4582      */
4583     public static GrouperInstallerUpgradeTaskAction valueOfIgnoreCase(String string, boolean exceptionIfBlank, boolean exceptionIfInvalid) {
4584       return GrouperInstallerUtils.enumValueOfIgnoreCase(GrouperInstallerUpgradeTaskAction.class, string, exceptionIfBlank, exceptionIfInvalid);
4585     }
4586     
4587   }
4588   
4589   /**
4590    * 
4591    */
4592   public static enum GrouperInstallerAdminManageService {
4593 
4594     /** tomcat */
4595     tomcat,
4596     
4597     /** database */
4598     database,
4599     
4600     /** daemon (loader) */
4601     grouperDaemon;
4602     
4603     /**
4604      * 
4605      * @param string
4606      * @param exceptionIfInvalid
4607      * @param exceptionIfBlank
4608      * @return the action
4609      */
4610     public static GrouperInstallerAdminManageService valueOfIgnoreCase(String string, boolean exceptionIfBlank, boolean exceptionIfInvalid) {
4611       return GrouperInstallerUtils.enumValueOfIgnoreCase(GrouperInstallerAdminManageService.class, string, exceptionIfBlank, exceptionIfInvalid);
4612     }
4613     
4614   }
4615   
4616   /**
4617    * 
4618    */
4619   public static enum GrouperInstallerManageAction {
4620 
4621     /** logs */
4622     logs,
4623 
4624     /** back to admin */
4625     back,
4626 
4627     /** exit */
4628     exit,
4629 
4630     /** services */
4631     services;
4632     
4633     /**
4634      * 
4635      * @param string
4636      * @param exceptionIfInvalid
4637      * @param exceptionIfBlank
4638      * @return the action
4639      */
4640     public static GrouperInstallerManageAction valueOfIgnoreCase(String string, boolean exceptionIfBlank, boolean exceptionIfInvalid) {
4641       return GrouperInstallerUtils.enumValueOfIgnoreCase(GrouperInstallerManageAction.class, string, exceptionIfBlank, exceptionIfInvalid);
4642     }
4643     
4644   }
4645   
4646   /**
4647    * 
4648    */
4649   public static enum GrouperInstallerDevelopAction {
4650 
4651     /** logs */
4652     translate,
4653 
4654     /** back to admin */
4655     back,
4656 
4657     /** exit */
4658     exit;
4659 
4660     /**
4661      * 
4662      * @param string
4663      * @param exceptionIfInvalid
4664      * @param exceptionIfBlank
4665      * @return the action
4666      */
4667     public static GrouperInstallerDevelopAction valueOfIgnoreCase(String string, boolean exceptionIfBlank, boolean exceptionIfInvalid) {
4668       return GrouperInstallerUtils.enumValueOfIgnoreCase(GrouperInstallerDevelopAction.class, string, exceptionIfBlank, exceptionIfInvalid);
4669     }
4670     
4671   }
4672   
4673   /**
4674    * get the existing properties file of patches
4675    * @return the file for patches
4676    */
4677   private Properties patchExistingProperties() {
4678     File patchExistingPropertiesFile = this.patchExistingPropertiesFile();
4679     if (patchExistingPropertiesFile == null || !patchExistingPropertiesFile.exists()) {
4680       return new Properties();
4681     }
4682     return GrouperInstallerUtils.propertiesFromFile(patchExistingPropertiesFile);
4683    }
4684 
4685   /**
4686    * get the existing properties file of patches
4687    * @return the file for patches
4688    */
4689   private File patchExistingPropertiesFile() {
4690     
4691     //dont cache this in a variable since the upgrade existing application variable
4692     File patchExistingPropertiesFile = null;
4693     //if theres a web-inf, put it there, if not, put it in regular...
4694     if (new File(this.upgradeExistingApplicationDirectoryString + "WEB-INF").exists()) {
4695       patchExistingPropertiesFile = new File(this.upgradeExistingApplicationDirectoryString 
4696           + "WEB-INF" + File.separator + "grouperPatchStatus.properties");
4697     } else {
4698       patchExistingPropertiesFile = new File(this.upgradeExistingApplicationDirectoryString 
4699           + "grouperPatchStatus.properties");
4700     }
4701     return patchExistingPropertiesFile;
4702   }
4703   
4704   /**
4705    * 
4706    */
4707   private void mainUpgradeLogic() {
4708 
4709     System.out.print("You should backup your files and database before you start.  Press <enter> to continue. ");
4710     readFromStdIn("grouperInstaller.autorun.backupFiles");
4711     
4712     System.out.println("\n##################################");
4713     System.out.println("Gather upgrade information\n");
4714 
4715     //####################################
4716     //Find out what directory to upgrade to.  This ends in a file separator
4717     this.grouperTarballDirectoryString = grouperUpgradeTempDirectory();
4718     
4719     //GRP-1429: grouperInstaller tarballs dir fails on upgrade
4720     //set this here since on upgrade some things download to this dir...
4721     this.grouperInstallDirectoryString = this.grouperTarballDirectoryString;
4722     
4723     //see what we are upgrading: api, ui, ws, client
4724     this.appToUpgrade = grouperAppToUpgradeOrPatch("upgrade");
4725 
4726     for (int i=0;i<10;i++) {
4727       System.out.println("Are there any running processes using this installation?  tomcats?  loader?  psp?  etc?  (t|f)? [f]:");
4728       boolean runningProcesses = readFromStdInBoolean(true, "grouperInstaller.autorun.runningProcesses");
4729       if (runningProcesses) {
4730         break;
4731       }
4732       System.out.println("Please stop any processes using this installation...");
4733       //lets sleep for a bit to let it start
4734       GrouperInstallerUtils.sleep(2000);
4735     }
4736     
4737     //get the directory where the existing installation is
4738     this.upgradeExistingApplicationDirectoryString = upgradeExistingDirectory();
4739 
4740     this.version = GrouperInstallerUtils.propertiesValue("grouper.version", true);
4741     System.out.println("Upgrading to grouper " + this.appToUpgrade.name() + " version: " + this.version);
4742  
4743     fixIndexFileIfOk();
4744     
4745     System.out.println("\n##################################");
4746     System.out.println("Download and build grouper packages\n");
4747 
4748     //download new files
4749     this.appToUpgrade.downloadAndBuildGrouperProjects(this);
4750 
4751     System.out.println("End download and build grouper packages\n");
4752     System.out.println("\n##################################");
4753 
4754     this.grouperBaseBakDir = this.grouperTarballDirectoryString + "bak_" + this.appToUpgrade + "_" 
4755         + new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss_SSS").format(new Date()) + File.separator; 
4756 
4757     GrouperInstallerUtils.tempFilePathForJars = this.grouperBaseBakDir 
4758         + "jarToDelete" + File.separator;
4759 
4760     //when we revert patches, default should be true
4761     this.revertAllPatchesDefault = true;
4762     try {
4763       this.appToUpgrade.upgradeApp(this);
4764     } finally {
4765       this.revertAllPatchesDefault = false;
4766     }
4767 
4768     this.reportOnConflictingJars(this.upgradeExistingApplicationDirectoryString);
4769     
4770     System.out.println("\nGrouper is upgraded from " + (this.originalGrouperJarVersion == null ? null : this.originalGrouperJarVersion) 
4771         + " to " + GrouperInstallerUtils.propertiesValue("grouper.version", true) +  "\n");
4772 
4773     //this file keeps track of partial upgrades
4774     GrouperInstallerUtils.fileDelete(this.grouperUpgradeOriginalVersionFile);
4775 
4776     //reset this so that patches go against new version
4777     this.grouperVersionOfJar = null;
4778     
4779   }
4780 
4781   /**
4782    * 
4783    */
4784   private void fixIndexFileIfOk() {
4785     Properties patchesExistingProperties = patchExistingProperties();
4786     
4787     //see what is already there
4788     String existingDate = patchesExistingProperties.getProperty("grouperInstallerLastFixedIndexFile.date");
4789 
4790     boolean defaultToFixIndex = true;
4791     
4792     if (!GrouperInstallerUtils.isBlank(existingDate)) {
4793       try {
4794         Date theDate = GrouperInstallerUtils.dateMinutesSecondsFormat.parse(existingDate);
4795         //this is when the installer was fixed to do the index file correctly
4796         if (theDate.getTime() > GrouperInstallerUtils.dateValue("20150929").getTime()) {
4797           defaultToFixIndex = false;
4798         }
4799       } catch (ParseException pe) {
4800         System.out.println("Cant parse date: " + existingDate);
4801       }
4802     }
4803 
4804     //if we are affecting 2.2.2+ then dont recommend this
4805     if (defaultToFixIndex) {
4806       
4807       //see the version
4808       String grouperVersion = this.grouperVersionOfJar().toString();
4809 
4810       GiGrouperVersion giGrouperVersion = GiGrouperVersion.valueOfIgnoreCase(grouperVersion);
4811       
4812       if (giGrouperVersion.greaterOrEqualToArg(GiGrouperVersion.valueOfIgnoreCase("2.2.2"))) {
4813         defaultToFixIndex = false;
4814       }
4815       
4816     }
4817     
4818     
4819     System.out.println("Do you want to fix the patch index file (download all patches and see if they are installed?) (" + (defaultToFixIndex ? "recommended" : "not recommended") + ") (t|f)? [" + (defaultToFixIndex ? "t" : "f") + "]: ");
4820     boolean fixIndexFile = readFromStdInBoolean(defaultToFixIndex, "grouperInstaller.autorun.fixIndexFile");
4821     if (fixIndexFile) {
4822       this.appToUpgrade.fixIndexFile(this);
4823     }
4824   }
4825 
4826   /**
4827    * upgrade the client
4828    */
4829   private void upgradeClient() {
4830 
4831     System.out.println("\n##################################");
4832     System.out.println("Upgrading grouper client\n");
4833 
4834     this.compareAndReplaceJar(this.grouperClientJar, new File(this.untarredClientDir + File.separator + "grouperClient.jar"), true, null);
4835 
4836     this.compareUpgradePropertiesFile(this.grouperClientBasePropertiesFile, 
4837       new File(this.untarredClientDir + File.separator + "grouper.client.base.properties"),
4838       this.grouperClientPropertiesFile,
4839       this.grouperClientExamplePropertiesFile,
4840       GrouperInstallerUtils.toSet("grouperClient.webService.client.version"),
4841       "grouperInstaller.autorun.removeRedundantPropetiesFromGrouperClient"
4842     );
4843 
4844     
4845   }
4846 
4847 
4848   /**
4849    * upgrade the ui
4850    */
4851   private void upgradeUi() {
4852 
4853     this.upgradeApiPreRevertPatch();
4854 
4855     System.out.println("You need to revert all patches to upgrade");
4856     this.patchRevertUi();
4857         
4858     System.out.println("\n##################################");
4859     System.out.println("Upgrading UI\n");
4860     
4861     //copy the jars there
4862     System.out.println("\n##################################");
4863     System.out.println("Upgrading UI jars\n");
4864 
4865     this.upgradeJars(new File(this.untarredUiDir + File.separator + "dist" + File.separator + "grouper" + File.separator 
4866         + "WEB-INF" + File.separator + "lib" + File.separator));
4867 
4868     System.out.println("\n##################################");
4869     System.out.println("Upgrading UI files\n");
4870 
4871     //copy files there
4872     this.copyFiles(this.untarredUiDir + File.separator + "dist" + File.separator + "grouper" + File.separator,
4873         this.upgradeExistingApplicationDirectoryString,
4874         GrouperInstallerUtils.toSet("WEB-INF/lib", "WEB-INF/web.xml", "WEB-INF/classes",
4875             "WEB-INF/bin/gsh", "WEB-INF/bin/gsh.bat", "WEB-INF/bin/gsh.sh"));
4876 
4877     {
4878       boolean hadChange = false;
4879       for (String gshName : new String[]{"gsh", "gsh.bat", "gsh.sh"}) {
4880         File newGshFile = new File(this.untarredUiDir + File.separator + "dist" 
4881             + File.separator + "grouper" + File.separator + "WEB-INF" + File.separator + "bin" 
4882             + File.separator + gshName);
4883 
4884         File existingGshFile = new File(this.upgradeExistingApplicationDirectoryString 
4885             + "WEB-INF" + File.separator + "bin" + File.separator + gshName);
4886 
4887         if (!GrouperInstallerUtils.contentEquals(newGshFile, existingGshFile)) {
4888           this.backupAndCopyFile(newGshFile, existingGshFile, true);
4889           if (!GrouperInstallerUtils.equals("gsh.bat", gshName)) {
4890             hadChange = true;
4891           }
4892         }
4893         
4894       }
4895       if (hadChange) {
4896         //set executable and dos2unix
4897         gshExcutableAndDos2Unix(this.untarredUiDir + File.separator + "dist" 
4898             + File.separator + "grouper" + File.separator + "WEB-INF" + File.separator + "bin" 
4899             + File.separator);
4900       }
4901     }
4902     
4903     upgradeWebXml(new File(this.untarredUiDir + File.separator + "dist" 
4904             + File.separator + "grouper" + File.separator + "WEB-INF" + File.separator + "web.xml"),
4905             new File(this.upgradeExistingApplicationDirectoryString 
4906                 + File.separator + "WEB-INF" + File.separator + "web.xml"));
4907     
4908     System.out.println("\n##################################");
4909     System.out.println("Upgrading UI config files\n");
4910 
4911     this.changeConfig("WEB-INF/classes/resources/grouper/nav.properties", 
4912         "WEB-INF/classes/grouperText/grouper.text.en.us.base.properties",
4913         "WEB-INF/classes/grouperText/grouper.text.en.us.properties", null, GiGrouperVersion.valueOfIgnoreCase("2.2.0"), true, 
4914         "grouperInstaller.autorun.continueAfterNavProperties",
4915         "grouperInstaller.autorun.removeOldKeysFromNavProperties");
4916 
4917     this.changeConfig("WEB-INF/classes/resources/grouper/media.properties", 
4918         "WEB-INF/classes/grouper-ui.base.properties",
4919         "WEB-INF/classes/grouper-ui.properties", null, GiGrouperVersion.valueOfIgnoreCase("2.2.0"), false,
4920         "grouperInstaller.autorun.continueAfterMediaProperties",
4921         "grouperInstaller.autorun.removeOldKeysFromMediaProperties");
4922 
4923     {
4924       File newBaseOwaspFile = new File(this.untarredUiDir + File.separator + "dist" 
4925           + File.separator + "grouper" + File.separator + "WEB-INF" + File.separator + "classes" 
4926           + File.separator + "Owasp.CsrfGuard.properties");
4927 
4928       File newOwaspFile = new File(this.untarredUiDir + File.separator + "dist" 
4929           + File.separator + "grouper" + File.separator + "WEB-INF" + File.separator + "classes" 
4930           + File.separator + "Owasp.CsrfGuard.overlay.properties");
4931 
4932       if (this.owaspCsrfGuardBaseFile == null) {
4933         this.owaspCsrfGuardBaseFile = new File(this.upgradeExistingClassesDirectoryString + newBaseOwaspFile.getName());
4934       }
4935       
4936       if (this.owaspCsrfGuardFile == null) {
4937         this.owaspCsrfGuardFile = new File(this.upgradeExistingClassesDirectoryString + newOwaspFile.getName());
4938       }
4939       
4940       this.backupAndCopyFile(newBaseOwaspFile, this.owaspCsrfGuardBaseFile, true);
4941 
4942       boolean editedOwaspOverlay = this.owaspCsrfGuardFile != null && this.owaspCsrfGuardFile.exists();
4943 
4944       File bakFile = this.backupAndCopyFile(newOwaspFile, this.owaspCsrfGuardFile, true);
4945 
4946       if (bakFile != null && editedOwaspOverlay) {
4947         if (!GrouperInstallerUtils.contentEquals(this.owaspCsrfGuardFile, newOwaspFile)) {
4948           System.out.println("If you have edited the Owasp.CsrfGuard.overlay.properties please merge the changes to the new file");
4949           System.out.println("Press <enter> when done");
4950           readFromStdIn("grouperInstaller.autorun.continueAfterEditedOwaspCsrfGuard");
4951         }
4952       }
4953     }
4954     
4955     this.upgradeApiPostRevertPatch();
4956 
4957     //patch it
4958     this.patchUi();
4959     
4960   }
4961 
4962   /**
4963    * upgrade the psp
4964    */
4965   private void upgradePsp() {
4966 
4967     this.upgradeApiPreRevertPatch();
4968 
4969     System.out.println("You need to revert all patches to upgrade");
4970     this.patchRevertPsp();
4971     
4972     System.out.println("\n##################################");
4973     System.out.println("Upgrading PSP\n");
4974     
4975     //copy the jars there
4976     System.out.println("\n##################################");
4977     System.out.println("Upgrading PSP jars\n");
4978 
4979     this.upgradeJars(new File(this.untarredPspDir + File.separator + "lib" + File.separator + "custom" + File.separator),
4980         new File(new File(this.upgradeExistingLibDirectoryString).getParentFile().getAbsolutePath() + File.separator + "custom"));
4981 
4982     System.out.println("\n##################################");
4983     System.out.println("Upgrading PSP files\n");
4984 
4985     //copy files there (this is the conf examples)
4986     this.copyFiles(this.untarredPspDir + File.separator + "conf" + File.separator,
4987         this.upgradeExistingApplicationDirectoryString + "conf" + File.separator, null);
4988 
4989     this.upgradeApiPostRevertPatch();
4990     
4991     //patch it
4992     this.patchPsp();
4993   }
4994 
4995   /**
4996    * upgrade the pspng
4997    */
4998   private void upgradePspng() {
4999 
5000     this.upgradeApiPreRevertPatch();
5001 
5002     System.out.println("You need to revert all patches to upgrade");
5003     this.patchRevertPspng();
5004     
5005     System.out.println("\n##################################");
5006     System.out.println("Upgrading PSPNG\n");
5007     
5008     //copy the jars there
5009     System.out.println("\n##################################");
5010     System.out.println("Upgrading PSPNG jars\n");
5011 
5012     this.upgradeJars(new File(this.untarredPspngDir + File.separator + "lib" + File.separator + "custom" + File.separator),
5013         new File(new File(this.upgradeExistingLibDirectoryString).getParentFile().getAbsolutePath() + File.separator + "custom"));
5014 
5015     System.out.println("\n##################################");
5016     System.out.println("Upgrading PSPNG files\n");
5017 
5018     //copy files there (this is the conf examples)
5019     this.copyFiles(this.untarredPspngDir + File.separator + "conf" + File.separator,
5020         this.upgradeExistingApplicationDirectoryString + "conf" + File.separator, null);
5021 
5022     this.upgradeApiPostRevertPatch();
5023     
5024     //patch it
5025     this.patchPspng();
5026   }
5027 
5028   
5029   /**
5030    * upgrade a web.xml file
5031    * @param newWebXml
5032    * @param existingWebXml
5033    */
5034   public void upgradeWebXml(File newWebXml, File existingWebXml) {
5035     
5036     File bakFile = backupAndCopyFile(newWebXml, existingWebXml, true);
5037     
5038     if (bakFile != null) {
5039       //it existed
5040       NodeList nodeList = GrouperInstallerUtils.xpathEvaluate(bakFile, "/web-app/security-constraint");
5041       boolean tookOutAuthn = false;
5042       if (nodeList == null || nodeList.getLength() == 0) {
5043         //take out authn from web.xml
5044         String webXmlContents = GrouperInstallerUtils.readFileIntoString(existingWebXml);
5045         int startAuthnIndex = webXmlContents.indexOf("<security-constraint>");
5046         int endAuthnIndex = webXmlContents.indexOf("</security-role>");
5047         if (startAuthnIndex != -1 && endAuthnIndex != -1 && endAuthnIndex > startAuthnIndex) {
5048           endAuthnIndex = endAuthnIndex + "</security-role>".length();
5049           //authn is there
5050           webXmlContents = webXmlContents.substring(0, startAuthnIndex) + webXmlContents.substring(endAuthnIndex, webXmlContents.length());
5051           GrouperInstallerUtils.saveStringIntoFile(existingWebXml, webXmlContents);
5052           tookOutAuthn = true;
5053           System.out.println("Taking out basic authentication from " + existingWebXml + " since it wasnt there before");
5054         }
5055       }
5056       System.out.println("If you customized the web.xml please merge your changes back in "
5057           + (tookOutAuthn ? "\n  Note: basic authentication was removed from the new web.xml to be consistent with the old web.xml" : "")
5058           + "\n  New file: " + existingWebXml.getAbsolutePath() + ", bak file:" + bakFile.getAbsolutePath() );
5059       System.out.println("Press the <enter> key to continue");
5060       readFromStdIn("grouperInstaller.autorun.continueAfterMergeWebXml");
5061       
5062       if (tookOutAuthn) {
5063         GrouperInstallerUtils.xpathEvaluate(existingWebXml, "/web-app");        
5064       }
5065     }
5066   }
5067 
5068   
5069   /**
5070    * @param legacyPropertiesFileRelativePath legacy file we are converting from
5071    * @param propertiesFileRelativePath 
5072    * @param propertiesToIgnore
5073    * @param basePropertiesFileRelativePath
5074    * @param versionMigrationHappened
5075    * @param removeOldCopy
5076    * @param autorunPropertiesKey key in properties file to automatically fill in a value
5077    * @param autorunPropertiesKeyRemoveOldKeys key in properties file to automatically fill in a value to remove old keys
5078    */
5079   @SuppressWarnings("unchecked")
5080   private void changeConfig(String legacyPropertiesFileRelativePath,
5081       String basePropertiesFileRelativePath,
5082       String propertiesFileRelativePath,
5083       Set<String> propertiesToIgnore,
5084       GiGrouperVersion versionMigrationHappened, boolean removeOldCopy, 
5085       String autorunPropertiesKey, String autorunPropertiesKeyRemoveOldKeys) {
5086 
5087     File legacyPropertiesFile = new File(this.upgradeExistingApplicationDirectoryString + legacyPropertiesFileRelativePath);
5088     File newBasePropertiesFile = new File(this.untarredUiDir + File.separator + "dist" 
5089         + File.separator + "grouper" + File.separator + basePropertiesFileRelativePath);
5090     File existingBasePropertiesFile = new File(this.upgradeExistingApplicationDirectoryString + basePropertiesFileRelativePath);
5091     File existingPropertiesFile = new File(this.upgradeExistingApplicationDirectoryString + propertiesFileRelativePath);
5092 
5093     this.compareUpgradePropertiesFile(existingBasePropertiesFile, newBasePropertiesFile, 
5094         existingPropertiesFile, null, propertiesToIgnore, autorunPropertiesKeyRemoveOldKeys);
5095 
5096     //look for existing properties in legacy file, and if there are properties in the base, then remove them
5097     if (legacyPropertiesFile.exists()) {
5098       
5099       Properties existingBaseProperties = GrouperInstallerUtils.propertiesFromFile(existingBasePropertiesFile);
5100       Properties existingProperties = GrouperInstallerUtils.propertiesFromFile(existingPropertiesFile);
5101       Properties legacyProperties = GrouperInstallerUtils.propertiesFromFile(legacyPropertiesFile);
5102       Set<String> propertyNamesToRemove = new LinkedHashSet<String>();
5103       Set<String> propertyNamesWrongValue = new LinkedHashSet<String>();
5104 
5105       for (String propertyName : (Set<String>)(Object)existingBaseProperties.keySet()) {
5106         if (legacyProperties.containsKey(propertyName)) {
5107           
5108           Object existingValue = existingProperties.containsKey(propertyName) ?
5109              existingProperties.get(propertyName) : existingBaseProperties.get(propertyName);
5110           
5111           //it might be in the override, what about other overrides?  who knows
5112           if (!GrouperInstallerUtils.equals(existingValue, 
5113               legacyProperties.get(propertyName))) {
5114 
5115             propertyNamesWrongValue.add(propertyName);
5116           }
5117           propertyNamesToRemove.add(propertyName);
5118         }
5119       }
5120       
5121       //if we found some, see if we can remove them
5122       if (propertyNamesToRemove.size() > 0) {
5123         
5124         if (propertyNamesWrongValue.size() > 0) {
5125 
5126           //these are properties that we different in the previous legacy file
5127           System.out.println(legacyPropertiesFileRelativePath + " has properties that have a different value than\n  the new place they are managed: "
5128               + basePropertiesFileRelativePath + ",\n  and the everride(s) which could be: " + propertiesFileRelativePath);
5129           System.out.println("Review these properties and merge the values, this could have happened due to changes in Grouper:");
5130           for (String propertyName: propertyNamesWrongValue) {
5131             System.out.println(" - " + propertyName);
5132           }
5133           System.out.println("When you are done merging press <enter>");
5134           readFromStdIn(autorunPropertiesKey);
5135 
5136         }
5137 
5138         if (removeOldCopy) {
5139           
5140           System.out.println(legacyPropertiesFileRelativePath + " is not used anymore by grouper, can it be backed up and removed (t|f)? [t]: ");
5141           boolean removeLegacy = readFromStdInBoolean(true, autorunPropertiesKeyRemoveOldKeys);
5142           if (removeLegacy) {
5143             File backupLegacy = bakFile(legacyPropertiesFile);
5144             GrouperInstallerUtils.copyFile(legacyPropertiesFile, backupLegacy, true);
5145             GrouperInstallerUtils.fileDelete(legacyPropertiesFile);
5146             System.out.println("File as removed.  Backup path: " + backupLegacy.getAbsolutePath());
5147           }
5148           
5149         } else {
5150           System.out.println(legacyPropertiesFileRelativePath + " has properties that can be removed since they are now managed in "
5151               + basePropertiesFileRelativePath);
5152           System.out.print("Would you like to have the properties automatically removed from " 
5153               + legacyPropertiesFile.getName() + " (t|f)? [t]: ");
5154           boolean removeRedundantProperties = readFromStdInBoolean(true, autorunPropertiesKeyRemoveOldKeys);
5155           
5156           if (removeRedundantProperties) {
5157             removeRedundantProperties(legacyPropertiesFile, propertyNamesToRemove);
5158           }
5159         }
5160       }
5161     }
5162   }
5163 
5164   
5165   /**
5166    * copy files if they are different from one place to another, print out statuses
5167    * @param fromDirString where to copy files from
5168    * @param toDirString where to copy files to
5169    * @param relativePathsToIgnore
5170    */
5171   public void copyFiles(String fromDirString, String toDirString, 
5172       Set<String> relativePathsToIgnore) {
5173     
5174     fromDirString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(fromDirString);
5175     toDirString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(toDirString);
5176 
5177     {
5178       //lets massage all the paths so they dont start or end with slash and
5179       //so they have File.separator instead of the wrong slash
5180       Set<String> tempRelativePathsToIgnore = new HashSet<String>();
5181       for (String path : GrouperInstallerUtils.nonNull(relativePathsToIgnore)) {
5182         path = GrouperInstallerUtils.fileMassagePathsNoLeadingOrTrailing(path);
5183         tempRelativePathsToIgnore.add(path);
5184       }
5185       relativePathsToIgnore = tempRelativePathsToIgnore;
5186     }
5187 
5188     int insertCount = 0;
5189     int updateCount = 0;
5190     Map<String, Boolean> relativeFilePathsChangedAndIfInsert = new LinkedHashMap<String, Boolean>();
5191 
5192     List<File> allFiles = GrouperInstallerUtils.fileListRecursive(new File(fromDirString));
5193     for (File fileToCopyFrom : allFiles) {
5194       String relativePath = null;
5195       {
5196         //get the relative path with no leading or trailing slash
5197         String path = fileToCopyFrom.getAbsolutePath();
5198         if (!GrouperInstallerUtils.filePathStartsWith(path,fromDirString)) {
5199           throw new RuntimeException("Why does path not start with fromDirString: " + path + ", " + fromDirString);
5200         }
5201         relativePath = path.substring(fromDirString.length());
5202         relativePath = GrouperInstallerUtils.fileMassagePathsNoLeadingOrTrailing(relativePath);
5203       }
5204       boolean ignore = false;
5205       
5206       //ignore paths passed in
5207       for (String pathToIgnore : relativePathsToIgnore) {
5208         if (GrouperInstallerUtils.filePathStartsWith(relativePath,pathToIgnore)) {
5209           ignore = true;
5210           break;
5211         }
5212       }
5213       
5214       if (!ignore) {
5215         
5216         //File to copy to
5217         File fileToCopyTo = new File(toDirString + relativePath);
5218         if (fileToCopyTo.exists()) {
5219           //compare contents
5220           if (GrouperInstallerUtils.contentEquals(fileToCopyFrom, fileToCopyTo)) {
5221             continue;
5222           }
5223           //not equals, make backup
5224           updateCount++;
5225 
5226           relativeFilePathsChangedAndIfInsert.put(relativePath, false);
5227 
5228           this.backupAndCopyFile(fileToCopyFrom, fileToCopyTo, false);
5229 
5230           continue;
5231         }
5232         
5233         //insert
5234         insertCount++;
5235         relativeFilePathsChangedAndIfInsert.put(relativePath, true);
5236         GrouperInstallerUtils.copyFile(fileToCopyFrom, fileToCopyTo, false);
5237         
5238       }
5239     }
5240 
5241     System.out.println("Upgrading files from: " + fromDirString + "\n  to: " + toDirString 
5242         + (GrouperInstallerUtils.length(relativePathsToIgnore) == 0 ? "" : 
5243         ("\n  ignoring paths: " + GrouperInstallerUtils.join(relativePathsToIgnore.iterator(), ", "))));
5244     System.out.println("Compared " + allFiles.size() + " files and found " 
5245         + insertCount + " adds and " + updateCount + " updates");
5246 
5247     if (insertCount > 0 || updateCount > 0) {
5248       
5249       System.out.println((insertCount + updateCount) + " files were backed up to: " + this.grouperBaseBakDir);
5250 
5251       boolean listFiles = insertCount + updateCount <= 10;
5252       if (!listFiles) {
5253         System.out.println("Do you want to see the list of files changed (t|f)? [f]: ");
5254         listFiles = readFromStdInBoolean(false, "grouperInstaller.autorun.viewListOfFilesChangedInCopy");
5255       }
5256 
5257       if (listFiles) {
5258 
5259         for (String relativeFilePathChanged : relativeFilePathsChangedAndIfInsert.keySet()) {
5260           boolean isInsert = relativeFilePathsChangedAndIfInsert.get(relativeFilePathChanged);
5261           System.out.println(relativeFilePathChanged + " was " + (isInsert ? "added" : "updated"));
5262         }
5263       }
5264     }
5265     
5266   }
5267 
5268   /**
5269    *
5270    * @param sourceDir directory to copy files from
5271    * @param targetDir directory to copy files to
5272    * @param filesToCopyFromSource list of files to copy, if exist in source and differ in target
5273    */
5274   private void syncFilesInDirWithBackup(String sourceDir, String targetDir, String[] filesToCopyFromSource) {
5275     for (String filename : filesToCopyFromSource) {
5276       File srcFile = new File(sourceDir + filename);
5277       File targetFile = new File(targetDir + filename);
5278 
5279       if (srcFile.isFile() && !GrouperInstallerUtils.contentEquals(srcFile, targetFile)) {
5280         try {
5281           @SuppressWarnings("unused")
5282           File bakFile = backupAndCopyFile(srcFile, targetFile, true);
5283         } catch (Exception e) {
5284           System.out.println(" - failed to copy newer bin file " + filename + ": " + e.getMessage());
5285         }
5286       }
5287     }
5288   }
5289 
5290   /**
5291    * upgrade the api
5292    */
5293   private void upgradeApi() {
5294     this.upgradeApiPreRevertPatch();
5295 
5296     System.out.println("You need to revert all patches to upgrade");
5297     this.patchRevertApi();
5298     
5299     this.upgradeApiPostRevertPatch();
5300   }
5301 
5302 
5303   /**
5304    * upgrade the api
5305    */
5306   private void upgradeApiPreRevertPatch() {
5307 
5308     //make sure existing gsh is executable and dos2unix
5309     gshExcutableAndDos2Unix(new File(gshCommand()).getParentFile().getAbsolutePath(), "existing");
5310     
5311     this.runChangeLogTempToChangeLog();
5312   }
5313   
5314   
5315   /**
5316    * upgrade the api
5317    */
5318   private void upgradeApiPostRevertPatch() {
5319 
5320     //revert patches
5321     this.upgradeClient();
5322 
5323     System.out.println("\n##################################");
5324     System.out.println("Upgrading API\n");
5325 
5326     //lets get the version of the existing jar
5327     this.originalGrouperJarVersionOrUpgradeFileVersion();
5328 
5329     this.compareAndReplaceJar(this.grouperJar, 
5330         new File(this.untarredApiDir + File.separator + "dist" + File.separator 
5331             + "lib" + File.separator + "grouper.jar"), true, null);
5332     
5333     if (this.appToUpgrade == AppToUpgrade.API) {
5334       boolean hadChange = false;
5335       for (String gshName : new String[]{"gsh", "gsh.bat", "gsh.sh"}) {
5336         File newGshFile = new File(this.untarredApiDir + File.separator + "bin" 
5337             + File.separator + gshName);
5338 
5339         File existingGshFile = new File(this.upgradeExistingApplicationDirectoryString 
5340             + "bin" + File.separator + gshName);
5341 
5342         if (!GrouperInstallerUtils.contentEquals(newGshFile, existingGshFile)) {
5343           this.backupAndCopyFile(newGshFile, existingGshFile, true);
5344           if (!GrouperInstallerUtils.equals("gsh.bat", gshName)) {
5345             hadChange = true;
5346           }
5347         }
5348         
5349       }
5350       if (hadChange) {
5351         //set executable and dos2unix
5352         gshExcutableAndDos2Unix(this.untarredApiDir + File.separator + "bin" 
5353             + File.separator);
5354       }
5355     }
5356 
5357     System.out.println("\n##################################");
5358     System.out.println("Upgrading API config files\n");
5359 
5360     this.compareUpgradePropertiesFile(this.grouperBasePropertiesFile, 
5361       new File(this.untarredApiDir + File.separator + "conf" + File.separator + "grouper.base.properties"),
5362       this.grouperPropertiesFile,
5363       this.grouperExamplePropertiesFile, null, 
5364       "grouperInstaller.autorun.removeRedundantPropetiesFromGrouperProperties"
5365     );
5366       
5367     this.compareUpgradePropertiesFile(this.grouperHibernateBasePropertiesFile, 
5368         new File(this.untarredApiDir + File.separator + "conf" + File.separator + "grouper.hibernate.base.properties"),
5369         this.grouperHibernatePropertiesFile,
5370         this.grouperHibernateExamplePropertiesFile, null, 
5371         "grouperInstaller.autorun.removeRedundantPropetiesFromGrouperHibernateProperties"
5372       );
5373         
5374     this.compareUpgradePropertiesFile(this.grouperLoaderBasePropertiesFile, 
5375         new File(this.untarredApiDir + File.separator + "conf" + File.separator + "grouper-loader.base.properties"),
5376         this.grouperLoaderPropertiesFile,
5377         this.grouperLoaderExamplePropertiesFile, null,
5378         "grouperInstaller.autorun.removeRedundantPropetiesFromGrouperLoaderProperties"
5379       );
5380     
5381     this.compareUpgradePropertiesFile(this.subjectBasePropertiesFile, 
5382         new File(this.untarredApiDir + File.separator + "conf" + File.separator + "subject.base.properties"),
5383         this.subjectPropertiesFile,
5384         null, null,
5385         "grouperInstaller.autorun.removeRedundantPropetiesFromSubjectProperties"
5386       );
5387     
5388     this.compareUpgradePropertiesFile(this.grouperCacheBasePropertiesFile, 
5389         new File(this.untarredApiDir + File.separator + "conf" + File.separator + "grouper.cache.base.properties"),
5390         this.grouperCachePropertiesFile,
5391         null, null,
5392         "grouperInstaller.autorun.removeRedundantPropetiesFromGrouperCacheProperties"
5393       );
5394 
5395     this.upgradeEhcacheXml();
5396     this.upgradeEhcacheXmlToProperties();
5397     this.upgradeSourcesXmlToProperties();
5398     
5399     this.compareAndCopyFile(this.grouperUtf8File, 
5400         new File(this.untarredApiDir + File.separator + "conf" + File.separator + "grouperUtf8.txt"),
5401         true,
5402         new File(this.upgradeExistingClassesDirectoryString)
5403         );
5404     
5405     this.compareAndCopyFile(this.gshFileLoadPropertiesFile, 
5406         new File(this.untarredApiDir + File.separator + "conf" + File.separator + "GSHFileLoad.properties"),
5407         true,
5408         new File(this.upgradeExistingClassesDirectoryString)
5409         );
5410     
5411     this.compareAndCopyFile(this.groovyshProfileFile, 
5412         new File(this.untarredApiDir + File.separator + "conf" + File.separator + "groovysh.profile"),
5413         true,
5414         new File(this.upgradeExistingClassesDirectoryString)
5415         );
5416     
5417     this.compareAndCopyFile(this.grouperClientUsageExampleFile, 
5418         new File(this.untarredApiDir + File.separator + "conf" + File.separator + "grouper.client.usage.example.txt"),
5419         true,
5420         new File(this.upgradeExistingClassesDirectoryString)
5421         );
5422     
5423     //do this only if less than 2.3.1
5424     if (new GiGrouperVersionersion(this.version).lessThanArg(new GiGrouperVersion("2.3.1"))) {
5425       System.out.println("\nYou should compare " + this.grouperPropertiesFile.getParentFile().getAbsolutePath() + File.separator + "sources.xml"
5426           + "\n  with " + this.untarredApiDir + File.separator + "conf" + File.separator + "sources.xml");
5427       System.out.print("Press <enter> to continue after you have merged the sources.xml. ");
5428       readFromStdIn("grouperInstaller.autorun.continueAfterMergingSourcesXml");
5429     }
5430     
5431     System.out.println("\n##################################");
5432     System.out.println("Upgrading API jars\n");
5433 
5434     this.upgradeJars(new File(this.untarredApiDir + File.separator + "lib" 
5435       + File.separator + "grouper" + File.separator));
5436     
5437     if (this.appToUpgrade.isApiOrganized()) {
5438 
5439       //if we need to put the jars in the jdbcSamples dir...
5440       this.upgradeJars(new File(this.untarredApiDir + File.separator + "lib" 
5441           + File.separator + "jdbcSamples" + File.separator), 
5442           new File(new File(this.upgradeExistingLibDirectoryString).getParentFile().getAbsolutePath() + File.separator + "jdbcSamples"));
5443     
5444     } else {
5445     
5446       this.upgradeJars(new File(this.untarredApiDir + File.separator + "lib" 
5447           + File.separator + "jdbcSamples" + File.separator));
5448     
5449     }
5450 
5451     {
5452       File subjectJar = new File(this.upgradeExistingLibDirectoryString + "subject.jar");
5453       if (subjectJar.exists()) {
5454         this.backupAndDeleteFile(subjectJar, true);
5455       }
5456     }
5457     
5458     {
5459       File oroJar = new File(this.upgradeExistingLibDirectoryString + "jakarta-oro.jar");
5460       if (oroJar.exists()) {
5461         this.backupAndDeleteFile(oroJar, true);
5462       }
5463     }
5464     
5465     System.out.println("\n##################################");
5466     System.out.println("Patch API\n");
5467 
5468     //patch it
5469     this.patchApi();
5470 
5471     //make sure log4j is debugging sql statements
5472     log4jDebugSql(this.upgradeExistingClassesDirectoryString + "log4j.properties");
5473     
5474     //verify that grouper.hibernate.properties doesn't have legacy properties
5475     removeLegacyHibernateProperties(this.upgradeExistingClassesDirectoryString + "grouper.hibernate.properties");
5476     
5477     System.out.println("\n##################################");
5478     System.out.println("Upgrading DB (registry)\n");
5479 
5480     this.apiUpgradeDbVersion(true);
5481 
5482     this.apiUpgradeAdditionalGshScripts();
5483 
5484   }
5485 
5486   /**
5487    * run additional GSH scripts based on what we are upgrading from...
5488    */
5489   private void apiUpgradeAdditionalGshScripts() {
5490     GiGrouperVersion giGrouperVersion = this.originalGrouperJarVersionOrUpgradeFileVersion();
5491     if (giGrouperVersion == null) {
5492       System.out.println("Grouper jar file: " + (this.grouperJar == null ? null : this.grouperJar.getAbsolutePath()));
5493       System.out.println("ERROR, cannot find grouper version in grouper jar file, do you want to continue? (t|f)? [f]: ");
5494       boolean continueScript = readFromStdInBoolean(false, "grouperInstaller.autorun.shouldContinueAfterNoGrouperVersionFound");
5495       if (!continueScript) {
5496         System.exit(1);
5497       }
5498     }
5499 
5500     boolean lessThan2_0 = this.originalGrouperJarVersion.lessThanArg(new GiGrouperVersion("2.0.0"));
5501     {
5502       String runUsduAutorun = null;
5503       if (lessThan2_0) {
5504         System.out.println("You are upgrading from pre API version 2.0.0, do you want to run Unresolvable Subject Deletion Utility (USDU) (recommended) (t|f)? [t]: ");
5505         runUsduAutorun = "grouperInstaller.autorun.runUsduPre2.0.0";
5506       } else {
5507         System.out.println("You are upgrading from after API version 2.0.0, so you dont have to do this,\n  "
5508             + "but do you want to run Unresolvable Subject Deletion Utility (USDU) (not recommended) (t|f)? [f]: ");
5509         runUsduAutorun = "grouperInstaller.autorun.runUsduPost2.0.0";
5510       }
5511       boolean runScript = readFromStdInBoolean(lessThan2_0, runUsduAutorun);
5512       
5513       if (runScript) {
5514         
5515         //running with command on command line doenst work on linux since the args with whitespace translate to 
5516         //save the commands to a file, and runt he file
5517         StringBuilder gshCommands = new StringBuilder();
5518   
5519         //gsh 0% GrouperSession.startRootSession()
5520         //edu.internet2.middleware.grouper.GrouperSession: 6f94c99d5b0948a3be96f94f00ab4d87,'GrouperSystem','application'
5521         //gsh 1% // run USDU to resolve all the subjects with type=person
5522         //gsh 3% usdu()
5523   
5524         gshCommands.append("grouperSession = GrouperSession.startRootSession();\n");
5525         gshCommands.append("usdu();\n");
5526   
5527         File gshFile = new File(this.untarredApiDir.getAbsolutePath() + File.separator + "gshUsdu.gsh");
5528         GrouperInstallerUtils.saveStringIntoFile(gshFile, gshCommands.toString());
5529         
5530         List<String> commands = new ArrayList<String>();
5531   
5532         addGshCommands(commands);
5533         commands.add(gshFile.getAbsolutePath());
5534   
5535         System.out.println("\n##################################");
5536         System.out.println("Running USDU with command:\n  " + convertCommandsIntoCommand(commands) + "\n");
5537   
5538         GrouperInstallerUtils.execCommand(
5539             GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
5540            new File(this.gshCommand()).getParentFile(), null, true);
5541   
5542       }
5543     }
5544     
5545     {
5546 
5547       String autorunResolveGroupSubjects = null;
5548       if (lessThan2_0) {
5549         System.out.println("You are upgrading from pre API version 2.0.0, do you want to resolve all group subjects (recommended) (t|f)? [t]: ");
5550         autorunResolveGroupSubjects = "grouperInstaller.autorun.resolveGroupSubjectsPre2.0.0";
5551       } else {
5552         System.out.println("You are upgrading from after API version 2.0.0, so you dont have to do this,\n  "
5553             + "but do you want to resolve all group subjects (not recommended) (t|f)? [f]: ");
5554         autorunResolveGroupSubjects = "grouperInstaller.autorun.resolveGroupSubjectsPost2.0.0";
5555       }
5556       boolean runScript = readFromStdInBoolean(lessThan2_0, autorunResolveGroupSubjects);
5557       
5558       if (runScript) {
5559         
5560         //running with command on command line doenst work on linux since the args with whitespace translate to 
5561         //save the commands to a file, and runt he file
5562         StringBuilder gshCommands = new StringBuilder();
5563   
5564         //gsh 5% GrouperSession.startRootSession();
5565         //edu.internet2.middleware.grouper.GrouperSession: 4163fb08b3b24922b55a14010d48e121,'GrouperSystem','application'
5566         //gsh 6% for (String g : HibernateSession.byHqlStatic().createQuery("select uuid from Group").listSet(String.class)) { subj = SubjectFinder.findByIdAndSource(g, "g:gsa", true); GrouperDAOFactory.getFactory().getMember().findBySubject(subj).updateMemberAttributes(subj, true); }
5567   
5568         gshCommands.append("grouperSession = GrouperSession.startRootSession();\n");
5569         gshCommands.append("for (String g : HibernateSession.byHqlStatic().createQuery(\"select uuid from Group\").listSet(String.class)) { subj = SubjectFinder.findByIdAndSource(g, \"g:gsa\", true); GrouperDAOFactory.getFactory().getMember().findBySubject(subj).updateMemberAttributes(subj, true); }\n");
5570   
5571         File gshFile = new File(this.untarredApiDir.getAbsolutePath() + File.separator + "gshGroupUsdu.gsh");
5572         GrouperInstallerUtils.saveStringIntoFile(gshFile, gshCommands.toString());
5573         
5574         List<String> commands = new ArrayList<String>();
5575   
5576         addGshCommands(commands);
5577         commands.add(gshFile.getAbsolutePath());
5578   
5579         System.out.println("\n##################################");
5580         System.out.println("Resolving group subjects with command:\n  " + convertCommandsIntoCommand(commands) + "\n");
5581 
5582         GrouperInstallerUtils.execCommand(
5583             GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
5584            new File(this.gshCommand()).getParentFile(), null, true);
5585       }
5586     }
5587 
5588     {
5589       boolean lessThan2_1 = giGrouperVersion.lessThanArg(new GiGrouperVersion("2.1.0"));
5590       String autorunSeeRuleCheckType = null;
5591       if (lessThan2_1) {
5592         System.out.println("You are upgrading from pre API version 2.1.0, do you want to "
5593             + "see if you have rules with ruleCheckType: flattenedPermission* (recommended) (t|f)? [t]: ");
5594         autorunSeeRuleCheckType = "grouperInstaller.autorun.seeRulesFlattenedPermissionsPre2.1.0";
5595       } else {
5596         System.out.println("You are upgrading from after API version 2.1.0, so you dont have to do this,\n  "
5597             + "but do you want to see if you have rules with ruleCheckType: flattenedPermission* (not recommended) (t|f)? [f]: ");
5598         autorunSeeRuleCheckType = "grouperInstaller.autorun.seeRulesFlattenedPermissionsPost2.1.0";
5599       }
5600       boolean runScript = readFromStdInBoolean(lessThan2_1, autorunSeeRuleCheckType);
5601       
5602       if (runScript) {
5603         
5604         //running with command on command line doenst work on linux since the args with whitespace translate to 
5605         //save the commands to a file, and runt he file
5606         StringBuilder gshCommands = new StringBuilder();
5607     
5608         gshCommands.append("\"Count: \" + HibernateSession.bySqlStatic().select(int.class, \"SELECT count(*) FROM grouper_rules_v WHERE rule_check_type LIKE 'flattenedPermission%'\");\n");
5609   
5610         File gshFile = new File(this.untarredApiDir.getAbsolutePath() + File.separator + "gshRuleFlattenedPermissionCount.gsh");
5611         GrouperInstallerUtils.saveStringIntoFile(gshFile, gshCommands.toString());
5612         
5613         List<String> commands = new ArrayList<String>();
5614   
5615         addGshCommands(commands);
5616         commands.add(gshFile.getAbsolutePath());
5617   
5618         System.out.println("\n##################################");
5619         System.out.println("Counting flattenedPermission rules with command:\n  " + convertCommandsIntoCommand(commands) + "\n");
5620 
5621         CommandResult commandResult = GrouperInstallerUtils.execCommand(
5622           GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
5623           new File(this.gshCommand()).getParentFile(), null, true);
5624 
5625         if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
5626           System.out.println("stderr: " + commandResult.getErrorText());
5627         }
5628         if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
5629           System.out.println("stdout: " + commandResult.getOutputText());
5630         }
5631 
5632         String result = commandResult.getOutputText().trim();
5633         String[] lines = GrouperInstallerUtils.splitLines(result);
5634         {
5635           Pattern pattern = Pattern.compile("^Count: ([0-9]+)$");
5636           int count = -1;
5637           for (String line : lines) {
5638             Matcher matcher = pattern.matcher(line);
5639             if (matcher.matches()) {
5640               count = GrouperInstallerUtils.intValue(matcher.group(1));
5641               break;
5642             }
5643           }
5644           if (count == -1) {
5645             System.out.println("Error getting count of rules, would you like to continue (t|f)? [t]:");
5646             if (!readFromStdInBoolean(true, "grouperInstaller.autorun.shouldContinueAfterErrorCountFlattenedRules")) {
5647               System.exit(1);
5648             }
5649           } else {
5650             if (count > 0) {
5651               System.out.println("You have " + count + " flattenedPermission rules that need to be removed.  You need to look in the view grouper_rules_v and notify the owners and remove these rules.  Do you want to continue (t|f)? [t]: ");
5652               
5653               if (!readFromStdInBoolean(true, "grouperInstaller.autorun.shouldContinueAfterFoundFlattenedRules")) {
5654                 System.exit(1);
5655               }
5656             }
5657           }
5658         }
5659       }
5660     }
5661 
5662     {
5663       boolean lessThan2_2_0 = giGrouperVersion.lessThanArg(new GiGrouperVersion("2.2.0"));
5664       String autorunRun2_2gshScript = null;
5665       if (lessThan2_2_0) {
5666         System.out.println("You are upgrading from pre API version 2.2.0, "
5667             + "do you want to run the 2.2 upgrade GSH script (recommended) (t|f)? [t]: ");
5668         autorunRun2_2gshScript = "grouperInstaller.autorun.run2.2gshUpgradeScriptPre2.2.0";
5669       } else {
5670         System.out.println("You are upgrading from after API version 2.2.0, so you dont have to do this,\n  "
5671             + "but do you want to run the 2.2 upgrade GSH script (not recommended) (t|f)? [f]: ");
5672         autorunRun2_2gshScript = "grouperInstaller.autorun.run2.2gshUpgradeScriptPost2.2.0";
5673       }
5674       boolean runScript = readFromStdInBoolean(lessThan2_2_0, autorunRun2_2gshScript);
5675       
5676       if (runScript) {
5677         
5678         File gshFile = new File(this.untarredApiDir.getAbsolutePath() + File.separator + "misc" + File.separator + "postGrouper2_2Upgrade.gsh");
5679         
5680         List<String> commands = new ArrayList<String>();
5681   
5682         addGshCommands(commands);
5683         commands.add(gshFile.getAbsolutePath());
5684   
5685         System.out.println("\n##################################");
5686         System.out.println("Running 2.2 upgrade GSH with command:\n  " + convertCommandsIntoCommand(commands) + "\n");
5687   
5688         GrouperInstallerUtils.execCommand(
5689             GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
5690            new File(this.gshCommand()).getParentFile(), null, true);
5691   
5692       }
5693       
5694     }
5695 
5696     {
5697       boolean lessThan2_2_1 = giGrouperVersion.lessThanArg(new GiGrouperVersion("2.2.1"));
5698       String autorunRun2_2_1gshUpgradeScript = null;
5699       if (lessThan2_2_1) {
5700         System.out.println("You are upgrading from pre API version 2.2.1, do you want to "
5701             + "run the 2.2.1 upgrade GSH script (recommended) (t|f)? [t]: ");
5702         autorunRun2_2_1gshUpgradeScript = "grouperInstaller.autorun.run2.2.1gshUpgradeScriptPre2.2.1";
5703       } else {
5704         System.out.println("You are upgrading from after API version 2.2.1, so you dont have to do this,\n  "
5705             + "but do you want to run the 2.2.1 upgrade GSH script (not recommended) (t|f)? [f]: ");
5706         autorunRun2_2_1gshUpgradeScript = "grouperInstaller.autorun.run2.2.1gshUpgradeScriptPost2.2.1";
5707       }
5708       boolean runScript = readFromStdInBoolean(lessThan2_2_1, autorunRun2_2_1gshUpgradeScript);
5709       
5710       if (runScript) {
5711         
5712         File gshFile = new File(this.untarredApiDir.getAbsolutePath() + File.separator + "misc" + File.separator + "postGrouper2_2_1Upgrade.gsh");
5713         
5714         List<String> commands = new ArrayList<String>();
5715 
5716         addGshCommands(commands);
5717         commands.add(gshFile.getAbsolutePath());
5718 
5719         System.out.println("\n##################################");
5720         System.out.println("Running 2.2.1 upgrade GSH with command:\n  " + convertCommandsIntoCommand(commands) + "\n");
5721 
5722         GrouperInstallerUtils.execCommand(
5723             GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
5724            new File(this.gshCommand()).getParentFile(), null, true);
5725 
5726       }
5727     }
5728 
5729     {
5730       boolean lessThan2_3_0 = giGrouperVersion.lessThanArg(new GiGrouperVersion("2.3.0"));
5731       String autorunRun2_3_0gshUpgradeScript = null;
5732       if (lessThan2_3_0) {
5733         System.out.println("You are upgrading from pre API version 2.3.0, do you want to "
5734             + "run the 2.3.0 upgrade GSH script (recommended) (t|f)? [t]: ");
5735         autorunRun2_3_0gshUpgradeScript = "grouperInstaller.autorun.run2.3.0gshUpgradeScriptPre2.3.0";
5736       } else {
5737         System.out.println("You are upgrading from after API version 2.3.0, so you dont have to do this,\n  "
5738             + "but do you want to run the 2.3.0 upgrade GSH script (not recommended) (t|f)? [f]: ");
5739         autorunRun2_3_0gshUpgradeScript = "grouperInstaller.autorun.run2.3.0gshUpgradeScriptPost2.3.0";
5740       }
5741       boolean runScript = readFromStdInBoolean(lessThan2_3_0, autorunRun2_3_0gshUpgradeScript);
5742       
5743       if (runScript) {
5744         
5745         File gshFile = new File(this.untarredApiDir.getAbsolutePath() + File.separator + "misc" + File.separator + "postGrouper2_3_0Upgrade.gsh");
5746         
5747         List<String> commands = new ArrayList<String>();
5748 
5749         addGshCommands(commands);
5750         commands.add(gshFile.getAbsolutePath());
5751 
5752         System.out.println("\n##################################");
5753         System.out.println("Running 2.3.0 upgrade GSH with command:\n  " + convertCommandsIntoCommand(commands) + "\n");
5754 
5755         GrouperInstallerUtils.execCommand(
5756             GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
5757            new File(this.gshCommand()).getParentFile(), null, true);
5758 
5759       }
5760     }
5761 
5762   }
5763 
5764   /**
5765    * grouper version of jar
5766    */
5767   private GiGrouperVersion grouperVersionOfJar = null;
5768   
5769   /**
5770    * 
5771    * @return the version
5772    */
5773   private GiGrouperVersion grouperVersionOfJar() {
5774     if (this.grouperVersionOfJar == null) {
5775       String grouperJarVersionString = null;
5776       if (this.grouperJar != null && this.grouperJar.exists()) {
5777         grouperJarVersionString = GrouperInstallerUtils.jarVersion(this.grouperJar);
5778         
5779       } else if (this.grouperClientJar != null && this.grouperClientJar.exists()) {
5780         grouperJarVersionString = GrouperInstallerUtils.jarVersion(this.grouperClientJar);
5781         
5782       }
5783       if (!GrouperInstallerUtils.isBlank(grouperJarVersionString)) {
5784         this.grouperVersionOfJar = new GiGrouperVersion(grouperJarVersionString);
5785       }
5786   
5787       if (this.grouperVersionOfJar == null) {
5788         throw new RuntimeException("Cant find version of grouper! " + this.grouperJar + ", " + this.grouperClientJar);
5789       }
5790     }
5791     
5792     return this.grouperVersionOfJar;
5793   }
5794   
5795   /**
5796    * grouper jar version e.g. 2.1.5
5797    */
5798   private GiGrouperVersion originalGrouperJarVersion = null;
5799   
5800   /**
5801    * keep trac if we have found it or not
5802    */
5803   private boolean originalGrouperJarVersionRetrieved = false;
5804   
5805   /**
5806    * get the version of the grouper jar
5807    * @return the version or null if couldnt be found
5808    */
5809   private GiGrouperVersion originalGrouperJarVersionOrUpgradeFileVersion() {
5810 
5811     if (!this.originalGrouperJarVersionRetrieved) {
5812 
5813       this.originalGrouperJarVersionRetrieved = true;
5814       
5815       //lets see if an upgrade went halfway through
5816       this.grouperUpgradeOriginalVersionFile = new File(this.upgradeExistingApplicationDirectoryString + "grouperUpgradeOriginalVersion.txt");
5817 
5818       this.originalGrouperJarVersion = this.grouperVersionOfJar();
5819       
5820       if (this.grouperUpgradeOriginalVersionFile.exists()) {
5821         String grouperJarVersionString = GrouperInstallerUtils.readFileIntoString(this.grouperUpgradeOriginalVersionFile);
5822         GiGrouperVersionrsion.html#GiGrouperVersion">GiGrouperVersion fileGrouperJarVersion = new GiGrouperVersion(grouperJarVersionString);
5823         
5824         if (fileGrouperJarVersion != this.originalGrouperJarVersion) {
5825           
5826           System.out.println("It is detected that an upgrade did not complete from version " + fileGrouperJarVersion);
5827           this.originalGrouperJarVersion = fileGrouperJarVersion;
5828         }
5829       } else {
5830         GrouperInstallerUtils.writeStringToFile(this.grouperUpgradeOriginalVersionFile, this.originalGrouperJarVersion.toString());
5831       }
5832     }
5833     
5834     return this.originalGrouperJarVersion;
5835   }
5836   
5837   /**
5838    * file where version is kept for partial upgrades
5839    */
5840   private File grouperUpgradeOriginalVersionFile;
5841   
5842   /**
5843    * @param firstTime if first time
5844    */
5845   private void apiUpgradeDbVersion(boolean firstTime) {
5846     
5847     if (!GrouperInstallerUtils.propertiesValueBoolean("grouperInstaller.default.api.checkDdlVersion", true, false)) {
5848       System.out.println("Not checking DDL version since grouper.installer.properties: grouperInstaller.default.api.checkDdlVersion = false");
5849       return;
5850     }
5851 
5852     List<String> commands = new ArrayList<String>();
5853 
5854     addGshCommands(commands);
5855     commands.add("-registry");
5856     commands.add("-check");
5857     commands.add("-noprompt");
5858 
5859     System.out.println("\n##################################");
5860     System.out.println("Checking API database version with command: " + convertCommandsIntoCommand(commands) + "\n");
5861 
5862     CommandResult commandResult = GrouperInstallerUtils.execCommand(
5863         GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
5864         new File(this.gshCommand()).getParentFile(), null, true);
5865     
5866     if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
5867       System.out.println("stdout: " + commandResult.getOutputText());
5868     }
5869     if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
5870       System.out.println("stderr: " + commandResult.getErrorText());
5871     }
5872 
5873     String result = commandResult.getErrorText().trim();
5874     
5875     // Grouper ddl object type 'Grouper' has dbVersion: 27 and java version: 28
5876     // NOTE: Grouper database schema DDL may require updates, but the temp change log must 
5877     // be empty to perform an upgrade.  To process the temp change log, start up your current 
5878     // version of GSH and run: loaderRunOneJob("CHANGE_LOG_changeLogTempToChangeLog")
5879     if (result != null && result.contains("CHANGE_LOG_changeLogTempToChangeLog")) {
5880       System.out.println("You must run the change log temp to change log before upgrading.  You can start the upgrader again and run it.");
5881       System.exit(1);
5882     }
5883     
5884     String[] lines = GrouperInstallerUtils.splitLines(result);
5885     {
5886       boolean okWithVersion = false;
5887       boolean notOkWithVersion = false;
5888       for (String line : lines) {
5889         line = line.toLowerCase();
5890         //expecting stderr: NOTE: database table/object structure (ddl) is up to date
5891         if (line.contains("ddl") && line.contains("up to date") && line.contains("note:")) {
5892           okWithVersion = true;
5893         }
5894         //cant have this line
5895         if (line.contains("requires updates")) {
5896           notOkWithVersion = true;
5897         }
5898       }
5899       if (okWithVersion && !notOkWithVersion) {
5900         return;
5901       }
5902     }
5903 
5904     if (!firstTime) {
5905       System.out.println("Error: we tried to upgrade the database but it didnt work, would you like to continue skipping DDL (t|f)? ");
5906       boolean continueOn = readFromStdInBoolean(null, "grouperInstaller.autorun.shouldContinueIfErrorUpgradingDatabase");
5907       if (continueOn) {
5908         return;
5909       }
5910     }
5911     
5912     //we need to upgrade the DDL
5913     //Grouper ddl object type 'Grouper' has dbVersion: 26 and java version: 28
5914     //Grouper database schema DDL requires updates
5915     //(should run script manually and carefully, in sections, verify data before drop statements, backup/export important data before starting, follow change log on confluence, dont run exact same script in multiple envs - generate a new one for each env),
5916     //script file is:
5917     //C:\app\grouper_2_2_0_installer\grouper.apiBinary-2.2.0\ddlScripts\grouperDdl_20141014_10_17_12_577.sql
5918     //Note: this script was not executed due to option passed in
5919     //To run script via gsh, carefully review it, then run this:
5920     //gsh -registry -runsqlfile C:\\app\\grouper_2_2_0_installer\\grouper.apiBinary-2.2.0\\ddlScripts\\grouperDdl_20141014_10_17_12_577.sql
5921 
5922     System.out.println("Review the script(s) above if there are any, do you want the upgrader to run it to upgrade the DDL for you (t|f)? [t]: ");
5923     boolean runIt = readFromStdInBoolean(true, "grouperInstaller.autorun.shouldRunDdlScript");
5924     
5925     if (runIt) {
5926 
5927       boolean foundScript = false;
5928 
5929       for (String line : lines) {
5930         if (line.contains("-registry -runsqlfile")) {
5931           
5932           String regexPattern = "^[^\\s]+\\s+-registry -runsqlfile (.*)$";
5933           Pattern pattern = Pattern.compile(regexPattern);
5934           
5935           Matcher matcher = pattern.matcher(line);
5936           
5937           if (!matcher.matches()) {
5938             throw new RuntimeException("Expected " + regexPattern + " but received: " + line);
5939           }
5940 
5941           String fileName = matcher.group(1);
5942           
5943           commands = new ArrayList<String>();
5944           
5945           addGshCommands(commands);
5946           commands.add("-registry");
5947           commands.add("-noprompt");
5948           commands.add("-runsqlfile");
5949           commands.add(fileName);
5950           
5951           foundScript = true;
5952           
5953           System.out.println("\n##################################");
5954           System.out.println("Upgrading database with command: " + convertCommandsIntoCommand(commands) + "\n");
5955 
5956           commandResult = GrouperInstallerUtils.execCommand(
5957               GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
5958              new File(this.gshCommand()).getParentFile(), null, true);
5959 
5960           //no out/err since printing as we go
5961           System.out.println("\nDone upgrading database");
5962           System.out.println("\n##################################\n");
5963         }
5964       }
5965       //cant find script, thats ok, just check and go
5966       if (!foundScript) {
5967         throw new RuntimeException("didnt find script to to run: " + result);
5968       }
5969 
5970       //check again to make sure ok
5971       apiUpgradeDbVersion(false);
5972     }
5973   }
5974 
5975   /**
5976    * upgrade jars from a directory
5977    * @param fromDir jars from this directory
5978    */
5979   private void upgradeJars(File fromDir) {
5980     this.upgradeJars(fromDir, new File(this.upgradeExistingLibDirectoryString));
5981   }
5982   
5983   /**
5984    * upgrade jars from a directory
5985    * @param fromDir jars from this directory
5986    * @param toDir is where jars go if not there
5987    */
5988   private void upgradeJars(File fromDir, File toDir) {
5989 
5990     //for each jar in the directory
5991     if (!fromDir.exists() || !fromDir.isDirectory()) {
5992       throw new RuntimeException("Why does jar directory not exist? " + fromDir);
5993     }
5994     
5995     int changes = 0;
5996     
5997     // sort the files to get them a little more reproducible
5998     File[] fromFiles = GrouperInstallerUtils.nonNull(fromDir.listFiles(), File.class);
5999     List<File> fromFilesList = GrouperInstallerUtils.toList(fromFiles);
6000     Collections.sort(fromFilesList);
6001     for (File jarFile : fromFilesList) {
6002       
6003       //only do jar files
6004       if (!jarFile.getName().endsWith(".jar")) {
6005         continue;
6006       }
6007       
6008       // File existingJar = this.findLibraryFile(jarFile.getName(), false);
6009 
6010       List<File> relatedJars = null;
6011       
6012       relatedJars = GrouperInstallerUtils.jarFindJar(toDir, jarFile.getName());
6013       
6014       boolean foundFile = false;
6015       if (GrouperInstallerUtils.length(relatedJars) > 0) {
6016         
6017         for (File relatedJar : relatedJars) {
6018           if (!relatedJar.exists()) {
6019             continue;
6020           }
6021           if (GrouperInstallerUtils.fileSha1(relatedJar).equals(GrouperInstallerUtils.fileSha1(jarFile))) {
6022             if (relatedJar.getName().equals(jarFile.getName())) {
6023               foundFile = true;
6024               continue;
6025             }
6026           }
6027           
6028           File bakFile = bakFile(relatedJar);
6029           
6030           System.out.println("Deleting " + relatedJar.getAbsolutePath() + ", backed up to: " + bakFile.getAbsolutePath());
6031           changes++;
6032           boolean moved = GrouperInstallerUtils.fileMove(relatedJar, bakFile, false);
6033           if (!moved) {
6034             System.out.println("Non-fatal error: could not delete file: " + relatedJar.getAbsolutePath() 
6035                 + ",\ndelete this file when all processed are terminated.  Press <enter> to acknowledge this.");
6036             readFromStdIn("grouperInstaller.autorun.continueAfterCantDeleteJar");
6037           }
6038         }
6039       }
6040       if (!foundFile) {
6041         changes += this.compareAndReplaceJar(null, jarFile, false, toDir) ? 1 : 0;
6042       }
6043           
6044       
6045 //      if (existingJar == null) {
6046 //        //see if one exists by another version
6047 //        if (GrouperInstallerUtils.length(relatedJars) == 1) {
6048 //          existingJar = relatedJars.get(0);
6049 //        }
6050 //      }
6051 
6052     }
6053 
6054     System.out.println("Upgraded " + changes + " jar files from: " + fromDir.getAbsolutePath()
6055         + "\n  to: " + toDir.getAbsolutePath());
6056     
6057   }
6058   
6059   /**
6060    * 
6061    */
6062   private void upgradeEhcacheXml() {
6063 
6064     //ehcache, prompt to see if do it (if difference than example, and if old example different than new example?
6065     File newEhcacheExample = new File(this.untarredApiDir + File.separator + "conf" + File.separator + "ehcache.xml");
6066 
6067     //this file is done
6068     if (!newEhcacheExample.exists() || this.ehcacheFile == null || !this.ehcacheFile.exists()) {
6069       return;
6070     }
6071     
6072     //lets see if different
6073     String existingEhcacheContents = GrouperInstallerUtils.readFileIntoString(this.ehcacheFile);
6074     String existingExampleEhcacheContents = GrouperInstallerUtils.readFileIntoString(this.ehcacheExampleFile);
6075     String newEhcacheContents = GrouperInstallerUtils.readFileIntoString(newEhcacheExample);
6076     
6077     //if existing is the same as new...
6078     if (GrouperInstallerUtils.equals(existingEhcacheContents, newEhcacheContents)) {
6079       //make sure example is up to date
6080       if (this.ehcacheExampleFile != null && !GrouperInstallerUtils.equals(existingExampleEhcacheContents, newEhcacheContents)) {
6081         this.backupAndCopyFile(newEhcacheExample, this.ehcacheExampleFile, true);
6082       }
6083 
6084       //we are all good
6085       return;
6086     }
6087 
6088     //lets backup the example and regular file
6089     File ehcacheBakFile = bakFile(this.ehcacheFile);
6090     GrouperInstallerUtils.copyFile(this.ehcacheFile, ehcacheBakFile, true);
6091 
6092     boolean mergeFiles = true;
6093     
6094     if (this.ehcacheExampleFile != null) {
6095       File ehcacheExampleBakFile = bakFile(this.ehcacheExampleFile);
6096   
6097       GrouperInstallerUtils.copyFile(this.ehcacheExampleFile, ehcacheExampleBakFile, true);
6098     } else {
6099       GrouperInstallerUtils.copyFile(newEhcacheExample, this.ehcacheFile, true);
6100       mergeFiles = false;
6101     }
6102 
6103     if (mergeFiles) {
6104       //if the ehcache is the same as the example, lets just copy
6105       if (GrouperInstallerUtils.equals(existingEhcacheContents, existingExampleEhcacheContents)) {
6106         this.backupAndCopyFile(newEhcacheExample, this.ehcacheFile, false);
6107         if (this.ehcacheExampleFile != null) {
6108           this.backupAndCopyFile(newEhcacheExample, this.ehcacheExampleFile, false);
6109         }
6110         return;
6111       }
6112 
6113       //the ehcache file is different from the example and different from the new one, so merge it in
6114       mergeEhcacheXmlFiles(newEhcacheExample, this.ehcacheExampleFile, this.ehcacheFile);
6115     }
6116 
6117     System.out.println("Compare you old ehcache.xml with the new ehcache.xml file: " 
6118         + "\n  Old file: "
6119         + ehcacheBakFile.getAbsolutePath()
6120         + "\n  New file: " + this.ehcacheFile.getAbsolutePath()
6121         + "\n  Press <enter> when done");
6122     readFromStdIn("grouperInstaller.autorun.continueAfterCompareEhcache");
6123 
6124   }
6125 
6126   /**
6127    * 
6128    */
6129   private void upgradeEhcacheXmlToProperties() {
6130 
6131     //dont do this is less than 2.3.1
6132     if (new GiGrouperVersionersion(this.version).lessThanArg(new GiGrouperVersion("2.3.1"))) {
6133       return;
6134     }
6135     
6136     // the file may have been added during the install
6137     if (this.grouperCacheBasePropertiesFile == null) {
6138       this.grouperCacheBasePropertiesFile = findClasspathFile("grouper.cache.base.properties", false);
6139     }
6140     
6141     //this file is done
6142     if ((this.ehcacheFile == null || !this.ehcacheFile.exists())
6143         && this.grouperCacheBasePropertiesFile.exists() && this.grouperCachePropertiesFile.exists()) {
6144       return;
6145     }
6146     
6147     System.out.print("Do you want to convert from ehcache.xml to grouper.cache.properties, note you need to do this to upgrade (t|f)? [t]: ");
6148     boolean convert = readFromStdInBoolean(true, "grouperInstaller.autorun.convertEhcacheXmlToProperties");
6149 
6150     if (!convert) {
6151       System.out.println("Note: grouper will not run, but whatever you want to do!!!!");
6152     }
6153     
6154     // the file may have been added during the install
6155     if (this.grouperCachePropertiesFile == null) {
6156       this.grouperCachePropertiesFile = findClasspathFile("grouper.cache.properties", false);
6157     }
6158     
6159     if (this.grouperCachePropertiesFile.exists()) {
6160       //see if there is anything in it
6161       Properties grouperCacheProperties = GrouperInstallerUtils.propertiesFromFile(this.grouperCachePropertiesFile);
6162       if (grouperCacheProperties.size() > 0) {
6163         this.backupAndDeleteFile(this.grouperCachePropertiesFile, true);
6164       } else {
6165         GrouperInstallerUtils.fileDelete(this.grouperCachePropertiesFile);
6166       }
6167     }
6168 
6169     URL ehcacheXmlUrl = null;
6170     
6171     try {
6172       ehcacheXmlUrl = this.ehcacheFile.toURI().toURL();
6173     } catch (Exception e) {
6174       throw new RuntimeException("Problem with ehcache.xml: " + (this.ehcacheFile == null ? null : this.ehcacheFile.getAbsoluteFile()), e);
6175     }
6176     
6177     //convert
6178     convertEhcacheXmlToProperties(this.grouperCacheBasePropertiesFile, this.grouperCachePropertiesFile, ehcacheXmlUrl);
6179     
6180     File bakFile = bakFile(this.grouperCachePropertiesFile);
6181     GrouperInstallerUtils.copyFile(this.grouperCachePropertiesFile, bakFile, true);
6182     this.backupAndDeleteFile(this.ehcacheFile, true);
6183     this.backupAndDeleteFile(this.ehcacheExampleFile, true);
6184     
6185   }
6186 
6187   /**
6188    * 
6189    * @param newFile
6190    * @param existingFile
6191    * @param printDetails
6192    * @return the bakFile
6193    */
6194   public File backupAndCopyFile(File newFile, File existingFile, boolean printDetails) {
6195     
6196     if (!GrouperInstallerUtils.contentEquals(newFile, existingFile)) {
6197       
6198       File bakFile = null;
6199           
6200       boolean fileExists = existingFile.exists();
6201       if (fileExists) {
6202         bakFile = bakFile(existingFile);
6203         GrouperInstallerUtils.copyFile(existingFile, bakFile, true);
6204         if (printDetails) {
6205           System.out.println("Backing up: " + existingFile.getAbsolutePath() + " to: " + bakFile.getAbsolutePath());
6206         }
6207       }
6208       if (printDetails) {
6209         System.out.println("Copying " + (fileExists ? "new file" : "upgraded file") + ": " + newFile.getAbsolutePath() + " to: " + existingFile.getAbsolutePath());
6210       }
6211       GrouperInstallerUtils.copyFile(newFile, existingFile, false);
6212       return bakFile;
6213       
6214     }
6215 
6216     if (printDetails) {
6217       System.out.println(existingFile.getAbsolutePath() + " has not been updated so it was not changed");
6218     }
6219     
6220     return null;
6221   }
6222 
6223   /**
6224    * @param file
6225    * @param printDetails
6226    * @return the bakFile
6227    */
6228   public File backupAndDeleteFile(File file, boolean printDetails) {
6229 
6230     if (file != null && file.exists()) {
6231 
6232       File bakFile = null;
6233 
6234       bakFile = bakFile(file);
6235       GrouperInstallerUtils.copyFile(file, bakFile, true);
6236       if (printDetails) {
6237         System.out.println("Backing up: " + file.getAbsolutePath() + " to: " + bakFile.getAbsolutePath());
6238       }
6239       if (printDetails) {
6240         System.out.println("Deleting file: " + file.getAbsolutePath());
6241       }
6242       GrouperInstallerUtils.fileDelete(file);
6243       return bakFile;
6244 
6245     }
6246 
6247     if (printDetails) {
6248       System.out.println(file + " did not exist so it was not deleted");
6249     }
6250 
6251     return null;
6252   }
6253 
6254   /**
6255    * 
6256    * @param existingFile
6257    * @return the bak file
6258    */
6259   public File bakFile(File existingFile) {
6260     String existingFilePath = existingFile.getAbsolutePath();
6261     if (!GrouperInstallerUtils.filePathStartsWith(existingFilePath, this.upgradeExistingApplicationDirectoryString)) {
6262       throw new RuntimeException("Why does existing path not start with upgrade path??? " 
6263           + existingFilePath + ", " + this.upgradeExistingApplicationDirectoryString);
6264     }
6265     
6266     String bakString = this.grouperBaseBakDir 
6267         + existingFilePath.substring(this.upgradeExistingApplicationDirectoryString.length());
6268 
6269     File bakFile = new File(bakString);
6270     return bakFile;
6271   }
6272   
6273   /**
6274    * @param existingBasePropertiesFile 
6275    * @param newBasePropertiesFile 
6276    * @param existingPropertiesFile 
6277    * @param existingExamplePropertiesFile 
6278    * @param propertiesToIgnore
6279    * @param autorunPropertiesKeyRemoveRedundantProperties key in properties file to automatically fill in a value
6280    */
6281   private void compareUpgradePropertiesFile(File existingBasePropertiesFile, 
6282       File newBasePropertiesFile,
6283       File existingPropertiesFile,
6284       File existingExamplePropertiesFile,
6285       Set<String> propertiesToIgnore, String autorunPropertiesKeyRemoveRedundantProperties) {
6286 
6287     boolean hadChange = false;
6288     
6289     if (!newBasePropertiesFile.exists() || !newBasePropertiesFile.isFile()) {
6290       throw new RuntimeException("Why does this file not exist? " + newBasePropertiesFile.getAbsolutePath());
6291     }
6292     
6293     //if there is an existing base properties file, compare and replace and done
6294     if (existingBasePropertiesFile != null && existingBasePropertiesFile.exists() && existingBasePropertiesFile.isFile()) {
6295       
6296       String existingBaseContents = GrouperInstallerUtils.readFileIntoString(existingBasePropertiesFile);
6297       String newBaseContents = GrouperInstallerUtils.readFileIntoString(newBasePropertiesFile);
6298       
6299       if (!GrouperInstallerUtils.equals(existingBaseContents, newBaseContents)) {
6300         
6301         String existingBasePropertiesFilePath = existingBasePropertiesFile.getAbsolutePath();
6302         if (!GrouperInstallerUtils.filePathStartsWith(existingBasePropertiesFilePath, this.upgradeExistingApplicationDirectoryString)) {
6303           throw new RuntimeException("Why does existing path not start with upgrade path??? " 
6304               + existingBasePropertiesFilePath + ", " + this.upgradeExistingApplicationDirectoryString);
6305         }
6306         
6307         String bakBasePropertiesString = this.grouperBaseBakDir + existingBasePropertiesFilePath.substring(this.upgradeExistingApplicationDirectoryString.length());
6308 
6309         File bakBasePropertiesFile = new File(bakBasePropertiesString);
6310         
6311         //make sure parents exist
6312         GrouperInstallerUtils.createParentDirectories(bakBasePropertiesFile);
6313 
6314         System.out.println(existingBasePropertiesFile.getName() + " has changes and was upgraded.\n  It is backed up to " 
6315             + bakBasePropertiesFile.getAbsolutePath());
6316         
6317         hadChange = true;
6318         
6319         GrouperInstallerUtils.fileMove(existingBasePropertiesFile, bakBasePropertiesFile);
6320         
6321         GrouperInstallerUtils.copyFile(newBasePropertiesFile, existingBasePropertiesFile, true);
6322 
6323       }
6324       
6325     } else {
6326       
6327       hadChange = true;
6328       
6329       System.out.println(newBasePropertiesFile.getName() + " didn't exist and was installed.");
6330       
6331       //its null, but we dont have the path...
6332       if (existingBasePropertiesFile == null) {
6333         existingBasePropertiesFile = new File(this.upgradeExistingClassesDirectoryString + newBasePropertiesFile.getName());
6334       }
6335       GrouperInstallerUtils.copyFile(newBasePropertiesFile, existingBasePropertiesFile, true);
6336     }
6337     
6338     // if there is an example there, it can be removed
6339     if (existingExamplePropertiesFile != null && existingExamplePropertiesFile.exists() && existingExamplePropertiesFile.isFile()) {
6340 
6341       String existingExamplePropertiesFilePath = existingExamplePropertiesFile.getAbsolutePath();
6342       if (!GrouperInstallerUtils.filePathStartsWith(existingExamplePropertiesFilePath, this.upgradeExistingApplicationDirectoryString)) {
6343         throw new RuntimeException("Why does existing path not start with upgrade path??? " 
6344             + existingExamplePropertiesFilePath + ", " + this.upgradeExistingApplicationDirectoryString);
6345       }
6346       
6347       String bakExamplePropertiesString = this.grouperBaseBakDir 
6348           + existingExamplePropertiesFilePath.substring(this.upgradeExistingApplicationDirectoryString.length());
6349 
6350       File bakExamplePropertiesFile = new File(bakExamplePropertiesString);
6351       
6352       //make sure parents exist
6353       GrouperInstallerUtils.createParentDirectories(bakExamplePropertiesFile);
6354 
6355       System.out.println(existingExamplePropertiesFile.getName() + " is not needed and was deleted.\n  It is backed up to " 
6356           + bakExamplePropertiesFile.getAbsolutePath());
6357 
6358       GrouperInstallerUtils.fileMove(existingExamplePropertiesFile, bakExamplePropertiesFile);
6359     
6360     }
6361 
6362     if (existingPropertiesFile != null && existingPropertiesFile.exists() && existingPropertiesFile.isFile()) {
6363       
6364       // now then, if there is a properties file, we can look for duplicate configs, and remove them...
6365       Set<String> duplicateConfigPropertyNames = configPropertyDuplicates(newBasePropertiesFile, existingPropertiesFile);
6366 
6367       if (GrouperInstallerUtils.length(propertiesToIgnore) > 0 && GrouperInstallerUtils.length(duplicateConfigPropertyNames) > 0) {
6368         duplicateConfigPropertyNames.addAll(propertiesToIgnore);
6369       }
6370       
6371       if (GrouperInstallerUtils.length(duplicateConfigPropertyNames) > 0) {
6372 
6373         hadChange = true;
6374         
6375         System.out.println(existingPropertiesFile.getName() + " has " + duplicateConfigPropertyNames.size() 
6376             + " properties that can be removed since the values are the same in "
6377             + newBasePropertiesFile.getName());
6378 
6379         System.out.println("Would you like to have the " + duplicateConfigPropertyNames.size() 
6380             + " redundant properties automatically removed from " 
6381             + existingPropertiesFile.getName() + " (t|f)? [t]: ");
6382         boolean removeRedundantProperties = readFromStdInBoolean(true, autorunPropertiesKeyRemoveRedundantProperties);
6383         
6384         if (removeRedundantProperties) {
6385 
6386           String existingPropertiesFilePath = existingPropertiesFile.getAbsolutePath();
6387           if (!GrouperInstallerUtils.filePathStartsWith(existingPropertiesFilePath, this.upgradeExistingApplicationDirectoryString)) {
6388             throw new RuntimeException("Why does existing path not start with upgrade path??? " 
6389                 + existingPropertiesFilePath + ", " + this.upgradeExistingApplicationDirectoryString);
6390           }
6391           
6392           String bakPropertiesString = this.grouperBaseBakDir 
6393               + existingPropertiesFilePath.substring(this.upgradeExistingApplicationDirectoryString.length());
6394 
6395           File bakPropertiesFile = new File(bakPropertiesString);
6396           
6397           //make sure parents exist
6398           GrouperInstallerUtils.createParentDirectories(bakPropertiesFile);
6399 
6400           System.out.println(existingPropertiesFile.getName() + " had redundant properties removed after being backed up to " 
6401               + bakPropertiesFile.getAbsolutePath());
6402 
6403           GrouperInstallerUtils.copyFile(existingPropertiesFile, bakPropertiesFile, true);
6404           
6405           removeRedundantProperties(existingPropertiesFile, duplicateConfigPropertyNames);
6406           
6407         }
6408       }
6409     } else {
6410       
6411       hadChange = true;
6412       
6413       //if we didnt have a properties file, create one
6414       //file is null...
6415       String contents = "\n# The " + newBasePropertiesFile.getName().replace(".base", "") 
6416           + " file uses Grouper Configuration Overlays (documented on wiki)\n"
6417           + "# By default the configuration is read from " + newBasePropertiesFile.getName() + "\n"
6418           + "# (which should not be edited), and the " +newBasePropertiesFile.getName().replace(".base", "") + " overlays\n"
6419           + "# the base settings.  See the " + newBasePropertiesFile.getName() + " for the possible\n"
6420           + "# settings that can be applied to the " + newBasePropertiesFile.getName().replace(".base", "") + "\n\n";
6421 
6422       File file = null;
6423       
6424       if (existingPropertiesFile != null) {
6425         file = existingPropertiesFile;
6426       } else if (existingBasePropertiesFile != null) {
6427         file = new File(existingBasePropertiesFile.getAbsolutePath().replace(".base", ""));
6428 //      } else {
6429 //        String fileName =  existingPropertiesFile != null ? existingPropertiesFile.getAbsolutePath() : this.upgradeExistingClassesDirectoryString + newBasePropertiesFile.getName().replace(".base", "");
6430 //        file = new File(fileName);
6431       }
6432       
6433       System.out.println("Created overlay config file: " + file.getAbsolutePath());
6434       
6435       GrouperInstallerUtils.saveStringIntoFile(file, contents);
6436     }
6437     
6438     if (!hadChange) {
6439       System.out.println("Found no changes in " + existingBasePropertiesFile.getAbsolutePath());
6440     }
6441     
6442   }
6443 
6444   /**
6445    * remove duplicate properties
6446    * @param propertiesFile
6447    * @param duplicatePropertyNames
6448    */
6449   private static void removeRedundantProperties(File propertiesFile, Set<String> duplicatePropertyNames) {
6450     
6451     String fileContents = GrouperInstallerUtils.readFileIntoString(propertiesFile);
6452     
6453     String newline = GrouperInstallerUtils.newlineFromFile(fileContents);
6454 
6455     StringBuilder newContents = new StringBuilder();
6456 
6457     String[] lines = GrouperInstallerUtils.splitLines(fileContents);
6458     
6459     boolean inStartComments = true;
6460     boolean inHeaderComments = false;
6461 
6462     StringBuilder captureHeader = new StringBuilder();
6463     StringBuilder propertyAndComments = new StringBuilder();
6464     
6465     for (String line: lines) {
6466       
6467       line = line.trim();
6468       
6469       boolean isBlank = GrouperInstallerUtils.isBlank(line);
6470       boolean isComment = line.startsWith("#");
6471       boolean isSingleComment = line.startsWith("#") && !line.startsWith("##");
6472       boolean isHeaderComment = line.contains("#####");
6473       boolean isProperty = !isBlank && !isComment;
6474       
6475       //if in header then we are done with the start comments
6476       if (isHeaderComment) {
6477         inStartComments = false;
6478       }
6479 
6480       //we want to keep the start comments
6481       if (inStartComments) {
6482         
6483         if (isBlank || isComment) {
6484           newContents.append(line).append(newline);
6485           continue;
6486         }
6487         inStartComments = false;
6488       }
6489 
6490       //we are done with headers
6491       if (isProperty || isBlank || isSingleComment) {
6492         inHeaderComments = false;
6493       }
6494 
6495       if (isHeaderComment) {
6496         //if header and in headers, then we arent in headers
6497         if (inHeaderComments) {
6498           inHeaderComments = false;
6499         } else {
6500           //if this is a header, and we arent in headers, then we are in headers
6501           inHeaderComments = true;          
6502           captureHeader.setLength(0);
6503         }
6504       }
6505 
6506       if (isHeaderComment || inHeaderComments) {
6507         propertyAndComments.setLength(0);
6508         captureHeader.append(line).append(newline);
6509         continue;
6510       }
6511       
6512       if (isProperty) {
6513         
6514         //get the property
6515         int equalsIndex = line.indexOf('=');
6516         if (equalsIndex == -1) {
6517           //uh... ignore this... 
6518           System.out.println("Invalid line removed from properties file: " + propertiesFile.getAbsolutePath() + ":\n  " + line);
6519           continue;
6520         }
6521         
6522         String propertyName = line.substring(0, equalsIndex).trim();
6523         //unescape colons...
6524         if (duplicatePropertyNames.contains(propertyName) || duplicatePropertyNames.contains(propertyName.replace("\\:", ":"))) {
6525           propertyAndComments.setLength(0);
6526           //remove it!
6527           continue;
6528         }
6529 
6530         //keep it
6531         propertyAndComments.append(line).append(newline);
6532 
6533         //we need a header if there is one
6534         if (captureHeader.length() > 0) {
6535           newContents.append(newline);
6536           newContents.append(captureHeader);
6537           captureHeader.setLength(0);
6538         }
6539 
6540         //append the property and contents
6541         newContents.append(propertyAndComments);
6542 
6543         propertyAndComments.setLength(0);
6544         continue;
6545       }
6546       
6547       //must be whitespace or comment...
6548       propertyAndComments.append(line).append(newline);
6549     }
6550     
6551     GrouperInstallerUtils.saveStringIntoFile(propertiesFile, newContents.toString());
6552     
6553   }
6554   
6555   /**
6556    * 
6557    * @param file1
6558    * @param file2
6559    * @return the property names which are the same
6560    */
6561   @SuppressWarnings("unchecked")
6562   public static Set<String> configPropertyDuplicates(File file1, File file2) {
6563     Properties file1properties = GrouperInstallerUtils.propertiesFromFile(file1);
6564     Properties file2properties = GrouperInstallerUtils.propertiesFromFile(file2);
6565     
6566     Set<String> duplicatePropertyNames = new LinkedHashSet<String>();
6567     
6568     for (String propertyName : (Set<String>)(Object)file2properties.keySet()) {
6569       
6570       String file1Value = GrouperInstallerUtils.trimToEmpty(file1properties.getProperty(propertyName));
6571       String file2Value = GrouperInstallerUtils.trimToEmpty(file2properties.getProperty(propertyName));
6572       
6573       if (GrouperInstallerUtils.equals(file1Value, file2Value)) {
6574         duplicatePropertyNames.add(propertyName);
6575       }
6576       
6577     }
6578     return duplicatePropertyNames;
6579   }
6580   
6581   
6582   /**
6583    * the location of the existing installation, must end in file separator
6584    */
6585   private String upgradeExistingApplicationDirectoryString;
6586   
6587   /**
6588    * 
6589    */
6590   private static enum AppToUpgrade {
6591     
6592     /**
6593      * upgrading the UI
6594      */
6595     UI {
6596 
6597       @Override
6598       public void patchStatus(GrouperInstaller grouperInstaller) {
6599         grouperInstaller.patchStatusUi();
6600       }
6601 
6602       @Override
6603       public void patch(GrouperInstaller grouperInstaller) {
6604         grouperInstaller.patchUi();
6605       }
6606 
6607       @Override
6608       public void revertPatch(GrouperInstaller grouperInstaller) {
6609         grouperInstaller.patchRevertUi();
6610       }
6611 
6612       @Override
6613       public boolean validateExistingDirectory(GrouperInstaller grouperInstaller) {
6614         //API and client are in the UI
6615         if (!API.validateExistingDirectory(grouperInstaller)) {
6616           return false;
6617         }
6618         
6619         //no need to check if it exists... its new in 2.2
6620 
6621         //grouperInstaller.mediaPropertiesFile = grouperInstaller.findClasspathFile("media.properties", false);
6622 
6623         //media should be there, but not forever
6624         
6625         return true;
6626       }
6627 
6628       @Override
6629       public void downloadAndBuildGrouperProjects(GrouperInstaller grouperInstaller) {
6630         API.downloadAndBuildGrouperProjects(grouperInstaller);
6631         
6632         //####################################
6633         //download and configure ui
6634         grouperInstaller.downloadAndConfigureUi();
6635 
6636         //####################################
6637         //get ant
6638         grouperInstaller.downloadAndUnzipAnt();
6639 
6640         //####################################
6641         //build UI
6642         grouperInstaller.buildUi(false);
6643 
6644         File serverXml = null;
6645         for (int i=0;i<10;i++) {
6646           String defaultServerXml = GrouperInstallerUtils.propertiesValue("grouperInstaller.default.ui.server.xml", false);
6647           System.out.println("What is the location of your tomcat server.xml for the UI?  "
6648               + "Note, if you dont use tomcat just leave it blank or type 'blank': " 
6649               + (GrouperInstallerUtils.isBlank(defaultServerXml) ? "" : ("[" + defaultServerXml + "]: ")));
6650           String serverXmlLocation = readFromStdIn("grouperInstaller.autorun.locationOfTomcatServerXml");
6651           
6652           if (GrouperInstallerUtils.equals(defaultServerXml, "blank")) {
6653             defaultServerXml = null;
6654             break;
6655           }
6656           
6657           if (GrouperInstallerUtils.isBlank(serverXmlLocation)) {
6658             if (GrouperInstallerUtils.isNotBlank(defaultServerXml)) {
6659               serverXmlLocation = defaultServerXml;
6660             } else {
6661               break;
6662             }
6663           }
6664           serverXml = new File(serverXmlLocation);
6665           if (serverXml.exists() && serverXml.isFile()) {
6666             break;
6667           }
6668           if (i != 9) {
6669             System.out.println("Error: server.xml cant be found, try again.");
6670           }
6671         }
6672         if (serverXml != null && serverXml.exists() && serverXml.isFile()) {
6673           grouperInstaller.configureTomcatUriEncoding(serverXml);
6674         }
6675                 
6676       }
6677 
6678       @Override
6679       public void upgradeApp(GrouperInstaller grouperInstaller) {
6680         grouperInstaller.upgradeUi();
6681       }
6682 
6683       @Override
6684       public void fixIndexFile(GrouperInstaller grouperInstaller) {
6685         grouperInstaller.fixIndexFileUi();
6686       }
6687 
6688       @Override
6689       public boolean isApiOrganized() {
6690         return false;
6691       }
6692     },
6693     
6694     /**
6695      * upgrading the API
6696      */
6697     API {
6698 
6699       @Override
6700       public boolean validateExistingDirectory(GrouperInstaller grouperInstaller) {
6701 
6702         //client is in the API
6703         if (!CLIENT.validateExistingDirectory(grouperInstaller)) {
6704           return false;
6705         }
6706         
6707         grouperInstaller.subjectPropertiesFile = grouperInstaller.findClasspathFile("subject.properties", false);
6708         grouperInstaller.subjectBasePropertiesFile = grouperInstaller.findClasspathFile("subject.base.properties", false);
6709 
6710         grouperInstaller.grouperUtf8File = grouperInstaller.findClasspathFile("grouperUtf8.txt", false);
6711         grouperInstaller.gshFileLoadPropertiesFile = grouperInstaller.findClasspathFile("GSHFileLoad.properties", false);
6712         grouperInstaller.grouperClientUsageExampleFile = grouperInstaller.findClasspathFile("grouper.client.usage.example.txt", false);
6713         grouperInstaller.groovyshProfileFile = grouperInstaller.findClasspathFile("groovysh.profile", false);
6714 
6715         //no need to check if it exists... its new in 2.2
6716         
6717         grouperInstaller.grouperPropertiesFile = grouperInstaller.findClasspathFile("grouper.properties", false);
6718         grouperInstaller.grouperBasePropertiesFile = grouperInstaller.findClasspathFile("grouper.base.properties", false);
6719         grouperInstaller.grouperExamplePropertiesFile = grouperInstaller.findClasspathFile("grouper.example.properties", false);
6720 
6721         if (grouperInstaller.grouperBasePropertiesFile == null 
6722             && grouperInstaller.grouperPropertiesFile == null 
6723             && grouperInstaller.grouperExamplePropertiesFile == null) {
6724           return false;
6725         }
6726         
6727         grouperInstaller.grouperHibernatePropertiesFile = grouperInstaller.findClasspathFile("grouper.hibernate.properties", false);
6728         grouperInstaller.grouperHibernateBasePropertiesFile = grouperInstaller.findClasspathFile("grouper.hibernate.base.properties", false);
6729         grouperInstaller.grouperHibernateExamplePropertiesFile = grouperInstaller.findClasspathFile("grouper.hibernate.example.properties", false);
6730 
6731         if (grouperInstaller.grouperHibernateBasePropertiesFile == null 
6732             && grouperInstaller.grouperHibernatePropertiesFile == null 
6733             && grouperInstaller.grouperHibernateExamplePropertiesFile == null) {
6734           return false;
6735         }
6736         
6737         grouperInstaller.grouperLoaderPropertiesFile = grouperInstaller.findClasspathFile("grouper-loader.properties", false);
6738         grouperInstaller.grouperLoaderBasePropertiesFile = grouperInstaller.findClasspathFile("grouper-loader.base.properties", false);
6739         grouperInstaller.grouperLoaderExamplePropertiesFile = grouperInstaller.findClasspathFile("grouper-loader.example.properties", false);
6740 
6741         if (grouperInstaller.grouperLoaderBasePropertiesFile == null 
6742             && grouperInstaller.grouperLoaderPropertiesFile == null 
6743             && grouperInstaller.grouperLoaderExamplePropertiesFile == null) {
6744           return false;
6745         }
6746         
6747         grouperInstaller.grouperCachePropertiesFile = grouperInstaller.findClasspathFile("grouper.cache.properties", false);
6748         grouperInstaller.grouperCacheBasePropertiesFile = grouperInstaller.findClasspathFile("grouper.cache.base.properties", false);
6749 
6750 //        //these must exist after 2.3.1+
6751 //        if (grouperInstaller.grouperCacheBasePropertiesFile == null 
6752 //            && grouperInstaller.grouperCachePropertiesFile == null 
6753 //            && new GiGrouperVersion(grouperInstaller.version).greaterOrEqualToArg(new GiGrouperVersion("2.3.1"))
6754 //            ) {
6755 //          return false;
6756 //        }
6757 
6758         //this must exist
6759         grouperInstaller.grouperJar = grouperInstaller.findLibraryFile("grouper.jar", false);
6760         if (grouperInstaller.grouperJar == null) {
6761           return false;
6762         }
6763 
6764         grouperInstaller.ehcacheFile = grouperInstaller.findClasspathFile("ehcache.xml", false);
6765         grouperInstaller.ehcacheExampleFile = grouperInstaller.findClasspathFile("ehcache.example.xml", false);        
6766         
6767         //all good
6768         return true;
6769       }
6770 
6771       @Override
6772       public void downloadAndBuildGrouperProjects(GrouperInstaller grouperInstaller) {
6773         CLIENT.downloadAndBuildGrouperProjects(grouperInstaller);
6774         
6775         //download api and set executable and dos2unix etc
6776         grouperInstaller.downloadAndConfigureApi();
6777 
6778       }
6779 
6780       @Override
6781       public void upgradeApp(GrouperInstaller grouperInstaller) {
6782         grouperInstaller.upgradeApi();
6783       }
6784 
6785       @Override
6786       public void patch(GrouperInstaller grouperInstaller) {
6787         grouperInstaller.patchApi();
6788       }
6789 
6790       @Override
6791       public void revertPatch(GrouperInstaller grouperInstaller) {
6792         grouperInstaller.patchRevertApi();
6793       }
6794 
6795       @Override
6796       public void patchStatus(GrouperInstaller grouperInstaller) {
6797         grouperInstaller.patchStatusApi();
6798       }
6799 
6800       @Override
6801       public void fixIndexFile(GrouperInstaller grouperInstaller) {
6802         grouperInstaller.fixIndexFileApi();
6803       }
6804 
6805       @Override
6806       public boolean isApiOrganized() {
6807         return true;
6808       }
6809     },
6810 
6811     /**
6812      * upgrading the client
6813      */
6814     CLIENT {
6815 
6816       @Override
6817       public void patchStatus(GrouperInstaller grouperInstaller) {
6818         throw new RuntimeException("Cant patch status client.  Client patches will be in the API if applicable");
6819       }
6820 
6821       @Override
6822       public void patch(GrouperInstaller grouperInstaller) {
6823         throw new RuntimeException("Cant patch client.  Client patches will be in the API if applicable");
6824       }
6825 
6826       @Override
6827       public void revertPatch(GrouperInstaller grouperInstaller) {
6828         throw new RuntimeException("Cant revert client.  Client patches will be in the API if applicable");
6829       }
6830 
6831       @Override
6832       public boolean validateExistingDirectory(GrouperInstaller grouperInstaller) {
6833 
6834         grouperInstaller.grouperClientPropertiesFile = grouperInstaller.findClasspathFile("grouper.client.properties", false);
6835         grouperInstaller.grouperClientBasePropertiesFile = grouperInstaller.findClasspathFile("grouper.client.base.properties", false);
6836         grouperInstaller.grouperClientExamplePropertiesFile = grouperInstaller.findClasspathFile("grouper.client.example.properties", false);
6837 
6838         if (grouperInstaller.grouperClientBasePropertiesFile == null 
6839             && grouperInstaller.grouperClientPropertiesFile == null 
6840             && grouperInstaller.grouperClientExamplePropertiesFile == null) {
6841           if (grouperInstaller.appToUpgrade == CLIENT) {
6842             return false;
6843           }
6844         }
6845         
6846         //this must exist
6847         grouperInstaller.grouperClientJar = grouperInstaller.findLibraryFile("grouperClient.jar", false);
6848         if (grouperInstaller.grouperClientJar == null) {
6849           if (grouperInstaller.appToUpgrade == CLIENT) {
6850             return false;
6851           }
6852         }
6853         
6854         //all good
6855         return true;
6856       }
6857 
6858       @Override
6859       public void downloadAndBuildGrouperProjects(GrouperInstaller grouperInstaller) {
6860         grouperInstaller.downloadAndBuildClient();
6861       }
6862 
6863       @Override
6864       public void upgradeApp(GrouperInstaller grouperInstaller) {
6865         grouperInstaller.upgradeClient();
6866       }
6867       
6868       @Override
6869       public void fixIndexFile(GrouperInstaller grouperInstaller) {
6870         throw new RuntimeException("Not implemented");
6871       }
6872 
6873       @Override
6874       public boolean isApiOrganized() {
6875         return false;
6876       }
6877     },
6878 
6879     /**
6880      * upgrading the WS
6881      */
6882     WS {
6883 
6884       @Override
6885       public void patchStatus(GrouperInstaller grouperInstaller) {
6886         grouperInstaller.patchStatusWs();
6887       }
6888 
6889       @Override
6890       public void patch(GrouperInstaller grouperInstaller) {
6891         grouperInstaller.patchWs();
6892       }
6893 
6894       @Override
6895       public void revertPatch(GrouperInstaller grouperInstaller) {
6896         grouperInstaller.patchRevertWs();
6897       }
6898 
6899       @Override
6900       public boolean validateExistingDirectory(GrouperInstaller grouperInstaller) {
6901         //API and client are in the UI
6902         if (!API.validateExistingDirectory(grouperInstaller)) {
6903           return false;
6904         }
6905 
6906         grouperInstaller.grouperWsPropertiesFile = grouperInstaller.findClasspathFile("grouper-ws.properties", false);
6907         grouperInstaller.grouperWsBasePropertiesFile = grouperInstaller.findClasspathFile("grouper-ws.base.properties", false);
6908         grouperInstaller.grouperWsExamplePropertiesFile = grouperInstaller.findClasspathFile("grouper-ws.example.properties", false);
6909 
6910         if (grouperInstaller.grouperWsBasePropertiesFile == null 
6911             && grouperInstaller.grouperWsPropertiesFile == null 
6912             && grouperInstaller.grouperWsExamplePropertiesFile == null) {
6913           return false;
6914         }
6915 
6916         return true;
6917       }
6918 
6919       @Override
6920       public void downloadAndBuildGrouperProjects(GrouperInstaller grouperInstaller) {
6921         API.downloadAndBuildGrouperProjects(grouperInstaller);
6922         
6923         //####################################
6924         //download and configure ws
6925         grouperInstaller.downloadAndUntarWs();
6926         
6927         //####################################
6928         //configure where api is
6929         grouperInstaller.configureWs();
6930 
6931         //####################################
6932         //get ant
6933         grouperInstaller.downloadAndUnzipAnt();
6934 
6935         //####################################
6936         //build Ws
6937         grouperInstaller.buildWs(false);
6938 
6939       }
6940 
6941       @Override
6942       public void upgradeApp(GrouperInstaller grouperInstaller) {
6943         grouperInstaller.upgradeWs();
6944       }
6945 
6946       @Override
6947       public void fixIndexFile(GrouperInstaller grouperInstaller) {
6948         grouperInstaller.fixIndexFileWs();
6949       }
6950 
6951       @Override
6952       public boolean isApiOrganized() {
6953         return false;
6954       }
6955     }, 
6956     
6957     /**
6958      * upgrading the UI
6959      */
6960     PSP {
6961 
6962       @Override
6963       public void patchStatus(GrouperInstaller grouperInstaller) {
6964         grouperInstaller.patchStatusPsp();
6965       }
6966 
6967       @Override
6968       public void patch(GrouperInstaller grouperInstaller) {
6969         grouperInstaller.patchPsp();
6970       }
6971 
6972       @Override
6973       public void revertPatch(GrouperInstaller grouperInstaller) {
6974         grouperInstaller.patchRevertPsp();
6975       }
6976 
6977       @Override
6978       public boolean validateExistingDirectory(GrouperInstaller grouperInstaller) {
6979         //API and client are in the UI
6980         if (!API.validateExistingDirectory(grouperInstaller)) {
6981           return false;
6982         }
6983         
6984         File customLibDir = new File(grouperInstaller.upgradeExistingApplicationDirectoryString + "lib" + File.separator + "custom");
6985         if (!customLibDir.exists()) {
6986           return false;
6987         }
6988 
6989         //see if psp jar is there
6990         List<File> files = GrouperInstallerUtils.jarFindJar(customLibDir, "psp.jar");
6991                 
6992         if (GrouperInstallerUtils.length(files) == 0) {
6993           return false;
6994         }
6995 
6996         return true;
6997       }
6998     
6999       @Override
7000       public void downloadAndBuildGrouperProjects(GrouperInstaller grouperInstaller) {
7001         API.downloadAndBuildGrouperProjects(grouperInstaller);
7002         
7003         //####################################
7004         //download and configure psp
7005         grouperInstaller.downloadAndBuildPsp();
7006     
7007       }
7008     
7009       @Override
7010       public void upgradeApp(GrouperInstaller grouperInstaller) {
7011         grouperInstaller.upgradePsp();
7012       }
7013 
7014       @Override
7015       public void fixIndexFile(GrouperInstaller grouperInstaller) {
7016         grouperInstaller.fixIndexFilePsp();
7017       }
7018       @Override
7019       public boolean isApiOrganized() {
7020         return true;
7021       }
7022     }, 
7023     
7024     /**
7025      * upgrading the UI
7026      */
7027     PSPNG {
7028     
7029       @Override
7030       public void patchStatus(GrouperInstaller grouperInstaller) {
7031         grouperInstaller.patchStatusPspng();
7032       }
7033     
7034       @Override
7035       public void patch(GrouperInstaller grouperInstaller) {
7036         grouperInstaller.patchPspng();
7037       }
7038     
7039       @Override
7040       public void revertPatch(GrouperInstaller grouperInstaller) {
7041         grouperInstaller.patchRevertPspng();
7042       }
7043     
7044       @Override
7045       public boolean validateExistingDirectory(GrouperInstaller grouperInstaller) {
7046         //API and client are in the UI
7047         if (!API.validateExistingDirectory(grouperInstaller)) {
7048           return false;
7049         }
7050         
7051         File customLibDir = new File(grouperInstaller.upgradeExistingApplicationDirectoryString + "lib" + File.separator + "custom");
7052         if (!customLibDir.exists()) {
7053           return false;
7054         }
7055     
7056         //see if psp jar is there
7057         String grouperVersion = grouperInstaller.grouperVersionOfJar().toString();
7058 
7059         List<File> files = GrouperInstallerUtils.jarFindJar(customLibDir, "grouper-pspng-" + grouperVersion + ".jar");
7060 
7061         if (GrouperInstallerUtils.length(files) == 0) {
7062           return false;
7063         }
7064     
7065         return true;
7066       }
7067     
7068       @Override
7069       public void downloadAndBuildGrouperProjects(GrouperInstaller grouperInstaller) {
7070         API.downloadAndBuildGrouperProjects(grouperInstaller);
7071         
7072         //####################################
7073         //download and configure psp
7074         grouperInstaller.downloadAndBuildPspng();
7075     
7076       }
7077     
7078       @Override
7079       public void upgradeApp(GrouperInstaller grouperInstaller) {
7080         grouperInstaller.upgradePspng();
7081       }
7082     
7083       @Override
7084       public void fixIndexFile(GrouperInstaller grouperInstaller) {
7085         grouperInstaller.fixIndexFilePspng();
7086       }
7087       @Override
7088       public boolean isApiOrganized() {
7089         return true;
7090       }
7091     };
7092 
7093     /**
7094      * if the organization is API organzied (e.g. has lib/jdbcSamples dir)
7095      * @return true/false
7096      */
7097     public abstract boolean isApiOrganized();
7098     
7099     /**
7100      * validate that the existing directory is valid, and find all the file paths
7101      * @param grouperInstaller 
7102      * @return true if valid, false if not
7103      */
7104     public abstract boolean validateExistingDirectory(GrouperInstaller grouperInstaller);
7105     
7106     /**
7107      * based on what is being upgraded, download and build the grouper projects
7108      * @param grouperInstaller
7109      */
7110     public abstract void downloadAndBuildGrouperProjects(GrouperInstaller grouperInstaller);
7111     
7112     /**
7113      * upgrade this app
7114      * @param grouperInstaller
7115      */
7116     public abstract void upgradeApp(GrouperInstaller grouperInstaller);
7117     
7118     /**
7119      * patch this app
7120      * @param grouperInstaller
7121      */
7122     public abstract void patch(GrouperInstaller grouperInstaller);
7123     
7124     /**
7125      * revert patch this app
7126      * @param grouperInstaller
7127      */
7128     public abstract void revertPatch(GrouperInstaller grouperInstaller);
7129     
7130     /**
7131      * patch status for this app
7132      * @param grouperInstaller
7133      */
7134     public abstract void patchStatus(GrouperInstaller grouperInstaller);
7135     
7136     /**
7137      * fix index file for this app
7138      * @param grouperInstaller
7139      */
7140     public abstract void fixIndexFile(GrouperInstaller grouperInstaller);
7141     
7142     /**
7143      * 
7144      * @param string
7145      * @param exceptionIfInvalid
7146      * @param exceptionIfBlank
7147      * @return the action
7148      */
7149     @SuppressWarnings("unused")
7150     public static AppToUpgrade valueOfIgnoreCase(String string, boolean exceptionIfBlank, boolean exceptionIfInvalid) {
7151       return GrouperInstallerUtils.enumValueOfIgnoreCase(AppToUpgrade.class, string, exceptionIfBlank, exceptionIfInvalid);
7152     }
7153 
7154   }
7155 
7156   /**
7157    * patch names installed (used for dependency checking), without the file ending e.g. grouper_v2_2_1_api_patch_0
7158    */
7159   private Set<String> patchesInstalled = new HashSet<String>();
7160   
7161   /**
7162    * if grouper is stopped
7163    */
7164   private boolean grouperStopped = false;
7165 
7166   /**
7167    * if should revert all
7168    */
7169   private Boolean revertAllPatches = null;
7170   
7171   /**
7172    * if we should use all local files
7173    */
7174   private Boolean useAllLocalFiles = null;
7175   
7176   /**
7177    * if we should use all unzipped files
7178    */
7179   private Boolean useAllUnzippedFiles = null;
7180   
7181   /**
7182    * if we should use all untarred directories
7183    */
7184   private Boolean useAllUntarredDirectories = null;
7185   
7186   /**
7187    * default for revert all patches
7188    */
7189   private boolean revertAllPatchesDefault = false;
7190   
7191   /**
7192    * if should revert all
7193    */
7194   private Boolean installAllPatches = null;
7195   
7196   /**
7197    * if should install some patches
7198    */
7199   private Boolean installPatchesUpToACertainPatchLevel = null;
7200   
7201   /**
7202    * if should install up to patch levels, comma separated
7203    * e.g. grouper_v2_3_0_api_patch_9, grouper_v2_3_0_ui_patch_10, grouper_v2_3_0_ws_patch_5
7204    */
7205   private String installPatchesUpToThesePatchLevels = null;
7206   
7207   /**
7208    * if should install certain specified
7209    */
7210   private Boolean installCertainSpecifiedPatches = null;
7211   
7212   /**
7213    * if should install up to patch levels, comma separated
7214    * e.g. grouper_v2_3_0_api_patch_0, grouper_v2_3_0_api_patch_1, grouper_v2_3_0_ui_patch_0
7215    */
7216   private String installCertainSpecifiedPatchesList = null;
7217   
7218   /**
7219    * if should revert certain specified
7220    */
7221   private Boolean revertCertainSpecifiedPatches = null;
7222   
7223   /**
7224    * if should revert up to patch levels, comma separated
7225    * e.g. grouper_v2_3_0_api_patch_0, grouper_v2_3_0_api_patch_1, grouper_v2_3_0_ui_patch_0
7226    */
7227   private String revertCertainSpecifiedPatchesList = null;
7228   
7229   /**
7230    * revert patches for an app
7231    * @param thisAppToRevert
7232    * @param originalAppToUpgrade 
7233    * @return if reverted
7234    */
7235   private boolean revertPatches(AppToUpgrade thisAppToRevert, AppToUpgrade originalAppToUpgrade) {
7236 
7237     if (thisAppToRevert == AppToUpgrade.CLIENT) {
7238       throw new RuntimeException("Cant revert " + thisAppToRevert);
7239     }
7240     
7241     Properties patchesExistingProperties = patchExistingProperties();
7242     
7243     String grouperVersion = this.grouperVersionOfJar().toString();
7244 
7245     grouperVersion = GrouperInstallerUtils.replace(grouperVersion, ".", "_");
7246 
7247     Map<Integer, String> patchNumberToNameBase = new LinkedHashMap<Integer, String>();
7248     
7249     boolean foundPatch = false;
7250 
7251     Map<String, Set<String>> installedPatchDependencies = new HashMap<String, Set<String>>();
7252     
7253     File patchExistingPropertiesFile = patchExistingPropertiesFile();
7254     
7255     for (int i=1000;i>=0;i--) {
7256       
7257       //grouper_v2_2_1_api_patch_0.state
7258       String keyBase = "grouper_v" + grouperVersion + "_" + thisAppToRevert.name().toLowerCase() + "_patch_" + i;
7259       String key = keyBase + ".state";
7260 
7261       patchNumberToNameBase.put(i, keyBase);
7262       
7263       String value = patchesExistingProperties.getProperty(key);
7264 
7265       if (!GrouperInstallerUtils.isBlank(value)) {
7266         
7267         System.out.println("\n################ Checking patch " + keyBase);
7268 
7269         GrouperInstallerPatchStatus grouperInstallerPatchStatus = GrouperInstallerPatchStatus.valueOfIgnoreCase(value, true, true);
7270         
7271         switch (grouperInstallerPatchStatus) {
7272           case skippedPermanently:
7273             
7274             System.out.println("Patch: " + keyBase + ": was skipped permanently on: " + patchesExistingProperties.getProperty(keyBase + ".date") + "\n");
7275             continue;
7276 
7277           case skippedTemporarily:
7278 
7279             System.out.println("Patch: " + keyBase + ": was skipped termporarily on: " + patchesExistingProperties.getProperty(keyBase + ".date") + "\n");
7280             continue;
7281 
7282           case reverted:
7283 
7284             System.out.println("Patch: " + keyBase + ": was removed on: " + patchesExistingProperties.getProperty(keyBase + ".date") + "\n");
7285             continue;
7286 
7287           case error:
7288 
7289             System.out.println("Patch: " + keyBase + ": had an error installing on: " + patchesExistingProperties.getProperty(keyBase + ".date") + "\n");
7290             continue;
7291 
7292           case applied:
7293             
7294             System.out.println("Patch: " + keyBase + ": was applied on: " + patchesExistingProperties.getProperty(keyBase + ".date")  + "\n");
7295             this.patchesInstalled.add(keyBase);
7296             break;
7297 
7298           default:
7299             throw new RuntimeException("Not expecting: " + grouperInstallerPatchStatus);
7300         }
7301 
7302       } else {
7303         continue;
7304       }
7305 
7306       if (!this.patchesInstalled.contains(keyBase)) {
7307         System.out.println("\n");
7308         continue;
7309       }
7310 
7311       //lets see if it exists on the server
7312       File patchUntarredDir = downloadAndUnzipPatch(keyBase);
7313       
7314       //if no more patches
7315       if (patchUntarredDir == null) {
7316         System.out.print("Error: cant find directory for patch: " + keyBase + ", press <enter> to continue. ");
7317         readFromStdIn("grouperInstaller.autorun.continueAfterCantFindPatchDir");
7318         continue;
7319       }
7320 
7321       //lets get the description:
7322       //  # will show up on screen so user knows what it is
7323       //  description = This patch fixes GRP-1080: browse folders refresh button only works in chrome, not other browsers
7324       //
7325       //  # patches that this patch is dependant on (comma separated)
7326       //  dependencies = 
7327       //
7328       //  # low, medium, or high risk to applying the patch
7329       //  risk = low
7330       //
7331       //  # is this is a security patch (true or false)
7332       //  security = false
7333       //
7334       //  # if this patch requires restart of processes (true or false)
7335       //  requiresRestart = false
7336       Properties patchProperties = GrouperInstallerUtils.propertiesFromFile(new File(patchUntarredDir.getAbsoluteFile() + File.separator + keyBase + ".properties"));
7337 
7338       foundPatch = true;
7339 
7340       // check dependencies
7341       {
7342         List<String> dependencies = GrouperInstallerUtils.splitTrimToList(patchProperties.getProperty("dependencies"), ",");
7343         Set<String> dependenciesSet = new HashSet<String>(GrouperInstallerUtils.nonNull(dependencies));
7344         installedPatchDependencies.put(keyBase, dependenciesSet);
7345       }
7346 
7347       boolean securityRelated = GrouperInstallerUtils.booleanValue(patchProperties.getProperty("security"), false);
7348       boolean requiresRestart = GrouperInstallerUtils.booleanValue(patchProperties.getProperty("requiresRestart"), true);
7349 
7350       if (this.revertAllPatches == null) {
7351         System.out.println("Would you like to revert all patches (t|f)? [" + (this.revertAllPatchesDefault ? "t" : "f") + "]: ");
7352         this.revertAllPatches = readFromStdInBoolean(this.revertAllPatchesDefault, "grouperInstaller.autorun.revertAllPatches");
7353       }
7354       
7355       if (!this.revertAllPatches && this.revertCertainSpecifiedPatches == null) {
7356         System.out.println("Would you like to revert certain specified patches? (t|f)? [f]: ");
7357         this.revertCertainSpecifiedPatches = readFromStdInBoolean(false, "grouperInstaller.autorun.revertCertainSpecifiedPatches");
7358 
7359         if (this.revertCertainSpecifiedPatches) {
7360 
7361           System.out.println("What patches would you like to revert [comma-separated] (e.g. grouper_v2_3_0_api_patch_0, grouper_v2_3_0_api_patch_1, grouper_v2_3_0_ui_patch_0)? : ");
7362           this.revertCertainSpecifiedPatchesList = readFromStdIn("grouperInstaller.autorun.revertCertainSpecifiedPatchesList");
7363         }
7364       }
7365       if (this.revertCertainSpecifiedPatches == null) {
7366         this.revertCertainSpecifiedPatches = false;
7367       }
7368 
7369       //print description
7370       System.out.println("Patch " + keyBase + " is " + patchProperties.getProperty("risk") + " risk, "
7371           + (securityRelated ? "is a security patch" : "is not a security patch"));
7372       System.out.println(patchProperties.getProperty("description"));
7373       
7374       Boolean revertPatch = null;
7375       
7376       if (this.revertAllPatches) {
7377         revertPatch = true;
7378       } else if (this.revertCertainSpecifiedPatches) {
7379         if (revertPatch == null) {
7380           revertPatch = shouldRevertCertainSpecifiedPatches(keyBase);
7381         }
7382       } else {
7383         System.out.print("Would you like to revert patch " + keyBase + " (t|f)? [f]: ");
7384         revertPatch = readFromStdInBoolean(false, "grouperInstaller.autorun.revertPatch");
7385       }
7386 
7387 
7388       if (!revertPatch) {
7389         System.out.println("");
7390         continue;
7391       }
7392 
7393       //check dependencies
7394       for (String patchName : installedPatchDependencies.keySet()) {
7395         
7396         Set<String> dependencies = GrouperInstallerUtils.nonNull(installedPatchDependencies.get(patchName));
7397         
7398         if (dependencies.contains(keyBase)) {
7399           System.out.println("Error: cant revert " + keyBase + " because an installed patch is dependent on it: " + patchName);
7400           System.exit(1);
7401         }
7402       }
7403 
7404       if (requiresRestart && !this.grouperStopped) {
7405         System.out.print("This patch requires all processes that user Grouper to be stopped.\n  "
7406             + "Please stop these processes if they are running and press <enter> to continue... ");
7407         this.grouperStopped = true;
7408         readFromStdIn("grouperInstaller.autorun.continueAfterStoppingGrouperProcesses");
7409       }
7410       
7411       Map<String, String> patchDirToApplicationPath = new LinkedHashMap<String, String>();
7412       patchDirToApplicationPath.put("files", this.upgradeExistingApplicationDirectoryString);
7413       patchDirToApplicationPath.put("classes", this.upgradeExistingClassesDirectoryString);
7414       patchDirToApplicationPath.put("lib", this.upgradeExistingLibDirectoryString);
7415       patchDirToApplicationPath.put("bin", this.upgradeExistingBinDirectoryString);
7416 
7417       boolean patchHasProblem = false;
7418       
7419       //we are reverting this patch, lets see if the files are there...
7420       //this.upgradeExistingApplicationDirectoryString
7421       //patchUntarredDir
7422       File newDir = new File(patchUntarredDir.getAbsolutePath() + File.separator + "new");
7423       File oldDir = new File(patchUntarredDir.getAbsolutePath() + File.separator + "old");
7424       {
7425 
7426         for (String patchDir : patchDirToApplicationPath.keySet()) {
7427 
7428           String applicationPath = patchDirToApplicationPath.get(patchDir);
7429 
7430           File newDirFiles = new File(newDir.getAbsoluteFile() + File.separator + patchDir);
7431           File oldDirFiles = new File(oldDir.getAbsoluteFile() + File.separator + patchDir);
7432           
7433           if (newDirFiles.exists() && newDirFiles.isDirectory()) {
7434 
7435             // relative, e.g. WEB-INF/jsp/someFile.jsp
7436             Set<String> newFileRelativePaths = GrouperInstallerUtils.fileDescendantRelativePaths(newDirFiles);
7437 
7438             for (String newFilePath : GrouperInstallerUtils.nonNull(newFileRelativePaths)) {
7439               File newFileInPatch = new File(newDirFiles.getAbsolutePath() + File.separator + newFilePath);
7440               File oldFileInPatch = new File(oldDirFiles.getAbsolutePath() + File.separator + newFilePath);
7441               
7442               if (revertPatchExcludes.contains(newFileInPatch.getName())) {
7443                 System.out.println("Skipping revert for file: " + newFileInPatch.getName());
7444                 continue;
7445               }
7446 
7447               File newFileInGrouper = new File(applicationPath + newFilePath);
7448 
7449               newFileInGrouper = fixLibraryFileIfFoundAndDifferent(newFileInGrouper, originalAppToUpgrade);
7450               
7451               if (!newFileInGrouper.exists() || !newFileInGrouper.isFile() 
7452                   || (!GrouperInstallerUtils.contentEquals(newFileInPatch, newFileInGrouper)
7453                       //its ok if the patch is already reverted?
7454                       && !GrouperInstallerUtils.contentEquals(oldFileInPatch, newFileInGrouper))) {
7455                 
7456                 // if it's just an example file and it didn't previously exist, then it's fine??
7457                 if (!newFileInGrouper.exists() && newFileInGrouper.getName().contains(".example.")) {
7458                   System.out.println("Grouper file " + newFileInGrouper.getAbsolutePath() + " doesn't exist.  Reverting patch anyways since this is an example file.");
7459                 } else {
7460                 
7461                   System.out.print("Problem reverting patch since this patch file:\n  " + newFileInPatch.getAbsolutePath() 
7462                       + "\n  is not the same as what the patch expects:\n  " + newFileInGrouper.getAbsolutePath()
7463                       + "\n  Do you want to force revert this patch (t|f)? [f]: ");
7464                   
7465                   boolean forceRevertPatch = readFromStdInBoolean(false, "grouperInstaller.autorun.forceRevertPatch");
7466                   
7467                   if (!forceRevertPatch) {
7468                     System.out.println("\nCannot revert patch since this patch file:\n  " + newFileInPatch.getAbsolutePath() 
7469                         + "\n  is not the same as what the patch expects:\n  " + newFileInGrouper.getAbsolutePath());
7470                     patchHasProblem = true;
7471                   }
7472                 }
7473               }
7474             }
7475           }
7476         }
7477       }
7478 
7479       {
7480         //deletes
7481         for (String patchDir : patchDirToApplicationPath.keySet()) {
7482 
7483           String applicationPath = patchDirToApplicationPath.get(patchDir);
7484 
7485           File newDirFiles = new File(newDir.getAbsoluteFile() + File.separator + patchDir);
7486           File oldDirFiles = new File(oldDir.getAbsoluteFile() + File.separator + patchDir);
7487           
7488           if (oldDirFiles.exists() && oldDirFiles.isDirectory()) {
7489 
7490             // relative, e.g. WEB-INF/jsp/someFile.jsp
7491             Set<String> oldFileRelativePaths = GrouperInstallerUtils.fileDescendantRelativePaths(oldDirFiles);
7492 
7493             for (String oldFilePath : GrouperInstallerUtils.nonNull(oldFileRelativePaths)) {
7494               File newFileInPatch = new File(newDirFiles.getAbsolutePath() + File.separator + oldFilePath);
7495               File oldFileInPatch = new File(oldDirFiles.getAbsolutePath() + File.separator + oldFilePath);
7496 
7497               //if there is a new file, then its not a delete
7498               if (newFileInPatch.exists()) {
7499                 continue;
7500               }
7501               
7502               if (revertPatchExcludes.contains(newFileInPatch.getName())) {
7503                 System.out.println("Skipping revert for file: " + newFileInPatch.getName());
7504                 continue;
7505               }
7506               
7507               File newFileInGrouper = new File(applicationPath + oldFilePath);
7508               
7509               newFileInGrouper = fixLibraryFileIfFoundAndDifferent(newFileInGrouper, originalAppToUpgrade);
7510 
7511               if (newFileInGrouper.exists() && newFileInGrouper.isFile() 
7512                   && !GrouperInstallerUtils.contentEquals(oldFileInPatch, newFileInGrouper)) {
7513                 
7514                 System.out.print("Problem reverting patch since this patch file:\n  " + oldFileInPatch.getAbsolutePath() 
7515                     + "\n  is not the same as what the patch expects (shouldnt exist):\n  " + newFileInGrouper.getAbsolutePath()
7516                     + "\n  Do you want to force revert this patch (t|f)? [f]: ");
7517                 
7518                 boolean forceRevertPatch = readFromStdInBoolean(true, "grouperInstaller.autorun.forceRevertPatch");
7519                 
7520                 if (!forceRevertPatch) {
7521                   System.out.println("\nCannot revert patch since this patch file:\n  " + newFileInPatch.getAbsolutePath() 
7522                       + "\n  is not the same as what the patch expects:\n  " + newFileInGrouper.getAbsolutePath());
7523                   patchHasProblem = true;
7524                 }
7525               }
7526             }
7527           }
7528         }
7529       }
7530 
7531       if (patchHasProblem) {
7532         System.out.println("Cannot continue since patch has problem");
7533         System.exit(1);
7534       }
7535       
7536       //so far so good, all the new files are all good, revert the patch
7537       for (String patchDir : patchDirToApplicationPath.keySet()) {
7538         
7539         String applicationPath = patchDirToApplicationPath.get(patchDir);
7540 
7541         File oldDirFiles = new File(oldDir.getAbsoluteFile() + File.separator + patchDir);
7542         File newDirFiles = new File(newDir.getAbsoluteFile() + File.separator + patchDir);
7543         
7544         if (newDirFiles.exists() && newDirFiles.isDirectory()) {
7545         
7546           // relative, e.g. WEB-INF/jsp/someFile.jsp
7547           Set<String> newFileRelativePaths = GrouperInstallerUtils.fileDescendantRelativePaths(newDirFiles);
7548           
7549           for (String newFilePath : GrouperInstallerUtils.nonNull(newFileRelativePaths)) {
7550 
7551             File oldFileInPatch = new File(oldDirFiles.getAbsolutePath() + File.separator + newFilePath);
7552 
7553             File newFileInGrouper = new File(applicationPath + newFilePath);
7554             
7555             if (revertPatchExcludes.contains(oldFileInPatch.getName())) {
7556               continue;
7557             }
7558             
7559             newFileInGrouper = fixLibraryFileIfFoundAndDifferent(newFileInGrouper, originalAppToUpgrade);
7560 
7561             if (oldFileInPatch.exists() && oldFileInPatch.isFile()) {
7562               System.out.println("Reverting file: " + newFileInGrouper.getAbsolutePath());
7563               GrouperInstallerUtils.copyFile(oldFileInPatch, newFileInGrouper, false);
7564             } else {
7565               System.out.println("Reverting (deleting) file: " + newFileInGrouper.getAbsolutePath());
7566               GrouperInstallerUtils.fileDelete(newFileInGrouper);
7567             }
7568           }
7569         }
7570       }
7571       
7572       //so far so good, revert the deletes
7573       for (String patchDir : patchDirToApplicationPath.keySet()) {
7574         
7575         String applicationPath = patchDirToApplicationPath.get(patchDir);
7576 
7577         File oldDirFiles = new File(oldDir.getAbsoluteFile() + File.separator + patchDir);
7578         File newDirFiles = new File(newDir.getAbsoluteFile() + File.separator + patchDir);
7579         
7580         if (oldDirFiles.exists() && oldDirFiles.isDirectory()) {
7581         
7582           // relative, e.g. WEB-INF/jsp/someFile.jsp
7583           Set<String> oldFileRelativePaths = GrouperInstallerUtils.fileDescendantRelativePaths(oldDirFiles);
7584           
7585           for (String oldFilePath : GrouperInstallerUtils.nonNull(oldFileRelativePaths)) {
7586 
7587             File oldFileInPatch = new File(oldDirFiles.getAbsolutePath() + File.separator + oldFilePath);
7588             File newFileInPatch = new File(newDirFiles.getAbsolutePath() + File.separator + oldFilePath);
7589 
7590             if (newFileInPatch.exists()) {
7591               continue;
7592             }
7593             
7594             if (revertPatchExcludes.contains(oldFileInPatch.getName())) {
7595               continue;
7596             }
7597             
7598             File newFileInGrouper = new File(applicationPath + oldFilePath);
7599             
7600             newFileInGrouper = fixLibraryFileIfFoundAndDifferent(newFileInGrouper, originalAppToUpgrade);
7601 
7602             if (oldFileInPatch.exists() && oldFileInPatch.isFile()) {
7603               System.out.println("Reverting deleted file: " + newFileInGrouper.getAbsolutePath());
7604               GrouperInstallerUtils.copyFile(oldFileInPatch, newFileInGrouper, false);
7605             }
7606           }
7607         }
7608       }
7609       
7610       //fixLibDir(patchDirToApplicationPath.get("lib"), originalAppToUpgrade);
7611       
7612       this.patchesInstalled.remove(keyBase);
7613       installedPatchDependencies.remove(keyBase);
7614       System.out.println("Patch successfully reverted: " + keyBase);
7615 
7616       editPropertiesFile(patchExistingPropertiesFile, keyBase + ".date", 
7617           GrouperInstallerUtils.dateMinutesSecondsFormat.format(new Date()), false);
7618       editPropertiesFile(patchExistingPropertiesFile, keyBase + ".state", 
7619           GrouperInstallerPatchStatus.reverted.name(), false);
7620 
7621       System.out.println("");
7622     }
7623 
7624     if (!foundPatch) {
7625       System.out.println("There are no new " + thisAppToRevert + " patches to revert\n");
7626       return false;
7627     }
7628     
7629     return true;
7630       
7631   }
7632   
7633   /**
7634    * this makes sure libs are in the right spot, though might be risky so dont do it
7635    * @param libDirWithSlash
7636    * @param originalAppToUpgrade
7637    */
7638   private void fixLibDir(String libDirWithSlash, AppToUpgrade originalAppToUpgrade) {
7639     if (originalAppToUpgrade.isApiOrganized()) {
7640       FilenameFilter apiFilenameFilter = new FilenameFilter() {
7641         
7642         public boolean accept(File dir, String name) {
7643           
7644           // any jars in "lib"
7645           if (GrouperInstallerUtils.equals("lib", dir.getName()) && name.endsWith(".jar")) {
7646             return true;
7647           }
7648           return false;
7649         }
7650       };
7651       //make sure all libs have something between lib and grouper
7652       for (File file : new File(libDirWithSlash).listFiles(apiFilenameFilter)) {
7653         // move this to grouper dir
7654         final File newFile = new File(file.getParentFile().getAbsolutePath() + File.separator + "grouper" + File.separator + file.getName());
7655         GrouperInstallerUtils.fileMove(file, newFile);
7656         System.out.println("Moving jar: " + file.getAbsolutePath() + " to " + newFile.getAbsolutePath());
7657       }
7658     } else {
7659       for (File file : GrouperInstallerUtils.fileListRecursive(new File(libDirWithSlash))) {
7660         // any jars not in "lib" but parent dir of dir is lib
7661         if (file.getName().endsWith(".jar") && !GrouperInstallerUtils.equals("lib", file.getParentFile().getName()) && GrouperInstallerUtils.equals("lib", file.getParentFile().getParentFile().getName())) {
7662           // move this to grouper dir
7663           final File newFile = new File(file.getParentFile().getParentFile().getAbsolutePath() + File.separator + file.getName());
7664           GrouperInstallerUtils.fileMove(file, newFile);
7665           System.out.println("Moving jar: " + file.getAbsolutePath() + " to " + newFile.getAbsolutePath());
7666           
7667         }
7668       }
7669     }
7670   }
7671   
7672   /**
7673    * get the patches available to apply that are not already applied
7674    * @param thisAppToUpgrade app to upgrade to check
7675    * @param originalAppToUpgrade 
7676    * @return if patches were installed
7677    */
7678   private boolean downloadAndInstallPatches(AppToUpgrade thisAppToUpgrade, AppToUpgrade originalAppToUpgrade) {
7679 
7680     if (thisAppToUpgrade == AppToUpgrade.CLIENT) {
7681       throw new RuntimeException("Cant install patches for " + thisAppToUpgrade);
7682     }
7683     
7684     Properties patchesExistingProperties = patchExistingProperties();
7685 
7686     String grouperVersion = GrouperInstallerUtils.propertiesValue("grouper.version", true);
7687 
7688     grouperVersion = GrouperInstallerUtils.replace(grouperVersion, ".", "_");
7689 
7690     Map<Integer, String> patchNumberToNameBase = new LinkedHashMap<Integer, String>();
7691     
7692     boolean foundNewPatch = false;
7693     
7694     File patchExistingPropertiesFile = patchExistingPropertiesFile();
7695     
7696     OUTER: for (int i=0;i<1000;i++) {
7697       
7698       //grouper_v2_2_1_api_patch_0.state
7699       String keyBase = "grouper_v" + grouperVersion + "_" + thisAppToUpgrade.name().toLowerCase() + "_patch_" + i;
7700       System.out.println("\n################ Checking patch " + keyBase);
7701       String key = keyBase + ".state";
7702 
7703       patchNumberToNameBase.put(i, keyBase);
7704       
7705       String value = patchesExistingProperties.getProperty(key);
7706 
7707       if (!GrouperInstallerUtils.isBlank(value)) {
7708         
7709         GrouperInstallerPatchStatus grouperInstallerPatchStatus = GrouperInstallerPatchStatus.valueOfIgnoreCase(value, true, true);
7710         
7711         switch (grouperInstallerPatchStatus) {
7712           case applied:
7713             
7714             System.out.println("Patch: " + keyBase + ": was applied on: " + patchesExistingProperties.getProperty(keyBase + ".date") + "\n");
7715             this.patchesInstalled.add(keyBase);
7716             
7717             continue;
7718 
7719           case skippedPermanently:
7720             
7721             System.out.println("Patch: " + keyBase + ": was skipped permanently on: " + patchesExistingProperties.getProperty(keyBase + ".date") + "\n");
7722             continue;
7723 
7724           case skippedTemporarily:
7725 
7726             System.out.println("Patch: " + keyBase + ": was skipped termporarily on: " + patchesExistingProperties.getProperty(keyBase + ".date") + "\n");
7727 
7728             break;
7729 
7730           case reverted:
7731 
7732             System.out.println("Patch: " + keyBase + ": was reverted on: " + patchesExistingProperties.getProperty(keyBase + ".date") + "\n");
7733 
7734             break;
7735 
7736           case error:
7737 
7738             System.out.println("Patch: " + keyBase + ": had an error installing on: " + patchesExistingProperties.getProperty(keyBase + ".date") + "\n");
7739 
7740             break;
7741 
7742           default:
7743             throw new RuntimeException("Not expecting: " + grouperInstallerPatchStatus);
7744         }
7745         
7746       }
7747 
7748       //lets see if it exists on the server
7749       File patchUntarredDir = downloadAndUnzipPatch(keyBase);
7750       
7751       //if no more patches
7752       if (patchUntarredDir == null) {
7753         System.out.println("");
7754         break OUTER;
7755       }
7756       
7757       //lets get the description:
7758       //  # will show up on screen so user knows what it is
7759       //  description = This patch fixes GRP-1080: browse folders refresh button only works in chrome, not other browsers
7760       //
7761       //  # patches that this patch is dependant on (comma separated)
7762       //  dependencies = 
7763       //
7764       //  # low, medium, or high risk to applying the patch
7765       //  risk = low
7766       //
7767       //  # is this is a security patch (true or false)
7768       //  security = false
7769       //
7770       //  # if this patch requires restart of processes (true or false)
7771       //  requiresRestart = false
7772       Properties patchProperties = GrouperInstallerUtils.propertiesFromFile(new File(patchUntarredDir.getAbsoluteFile() + File.separator + keyBase + ".properties"));
7773 
7774       foundNewPatch = true;
7775 
7776       Boolean installPatch = null;
7777       
7778       if (this.installPatchesUpToACertainPatchLevel != null && this.installPatchesUpToACertainPatchLevel) {
7779         if (!GrouperInstallerUtils.isBlank(this.installPatchesUpToThesePatchLevels)) {
7780           
7781           installPatch = shouldInstallPatchUpToLevel(keyBase);
7782           
7783           if (!installPatch) {
7784             break OUTER;
7785           }
7786         }
7787       }
7788       if (this.installCertainSpecifiedPatches != null && this.installCertainSpecifiedPatches) {
7789         if (!GrouperInstallerUtils.isBlank(this.installCertainSpecifiedPatchesList)) {
7790           
7791           installPatch = shouldInstallCertainSpecifiedPatches(keyBase);
7792           
7793         }
7794       }
7795      
7796       // check dependencies
7797       if (installPatch == null || installPatch == true){
7798         String[] dependencies = GrouperInstallerUtils.splitTrim(patchProperties.getProperty("dependencies"), ",");
7799   
7800         boolean invalidDependency = false;
7801         for (String dependency : GrouperInstallerUtils.nonNull(dependencies, String.class)) {
7802           if (!this.patchesInstalled.contains(dependency)) {
7803             System.out.println("Cannot install patch " + keyBase + " since it is dependent on a patch which is not installed: " + dependency);
7804             invalidDependency = true;
7805           }
7806         }
7807         if (invalidDependency) {
7808           System.out.println("Press <enter> to continue. ");
7809           readFromStdIn("grouperInstaller.autorun.continueAfterPatchDependencyFails");
7810           continue OUTER;
7811         }
7812       }
7813       
7814       boolean securityRelated = GrouperInstallerUtils.booleanValue(patchProperties.getProperty("security"), false);
7815       boolean requiresRestart = GrouperInstallerUtils.booleanValue(patchProperties.getProperty("requiresRestart"), true);
7816       
7817       if (this.installAllPatches == null) {
7818         System.out.println("Would you like to install all patches (t|f)? [t]: ");
7819         this.installAllPatches = readFromStdInBoolean(true, "grouperInstaller.autorun.installAllPatches");
7820 
7821         if (!this.installAllPatches && this.installPatchesUpToACertainPatchLevel == null ) {
7822           System.out.println("Would you like to install patches up to a certain patch level? (t|f)? [f]: ");
7823           this.installPatchesUpToACertainPatchLevel = readFromStdInBoolean(false, "grouperInstaller.autorun.installPatchesUpToACertainPatchLevel");
7824           
7825           if (this.installPatchesUpToACertainPatchLevel) {
7826 
7827             System.out.println("What patch levels would you like to install up to and including [comma-separated] (e.g. grouper_v2_3_0_api_patch_9, grouper_v2_3_0_ui_patch_10, grouper_v2_3_0_ws_patch_5)? : ");
7828             this.installPatchesUpToThesePatchLevels = readFromStdIn("grouperInstaller.autorun.installPatchesUpToThesePatchLevels");
7829 
7830           }
7831           
7832         }
7833         
7834         if (this.installPatchesUpToACertainPatchLevel == null) {
7835           this.installPatchesUpToACertainPatchLevel = false;
7836         }
7837         
7838         if (!this.installAllPatches && !this.installPatchesUpToACertainPatchLevel && this.installCertainSpecifiedPatches == null) {
7839           System.out.println("Would you like to install certain specified patches? (t|f)? [f]: ");
7840           this.installCertainSpecifiedPatches = readFromStdInBoolean(false, "grouperInstaller.autorun.installCertainSpecifiedPatches");
7841 
7842           if (this.installCertainSpecifiedPatches) {
7843 
7844             System.out.println("What patches would you like to install [comma-separated] (e.g. grouper_v2_3_0_api_patch_0, grouper_v2_3_0_api_patch_1, grouper_v2_3_0_ui_patch_0)? : ");
7845             this.installCertainSpecifiedPatchesList = readFromStdIn("grouperInstaller.autorun.installCertainSpecifiedPatchesList");
7846           }
7847         }
7848         if (this.installCertainSpecifiedPatches == null) {
7849           this.installCertainSpecifiedPatches = false;
7850         }
7851       }
7852 
7853       //print description
7854       System.out.println("Patch " + keyBase + " is " + patchProperties.getProperty("risk") + " risk, "
7855           + (securityRelated ? "is a security patch" : "is not a security patch"));
7856       System.out.println(patchProperties.getProperty("description"));
7857 
7858       if (this.installAllPatches) {
7859         installPatch = true;
7860       } else if (this.installPatchesUpToACertainPatchLevel) {
7861         if (installPatch == null) {
7862           installPatch = shouldInstallPatchUpToLevel(keyBase);
7863         }
7864       } else if (this.installCertainSpecifiedPatches) {
7865         if (installPatch == null) {
7866           installPatch = shouldInstallCertainSpecifiedPatches(keyBase);
7867         }
7868       } else {
7869         System.out.println("Would you like to install patch " + keyBase + " (t|f)? [t]: ");
7870         installPatch = readFromStdInBoolean(true, "grouperInstaller.autorun.installPatch");
7871       }
7872 
7873       //keep track that we skipped this in the patch properties file
7874       editPropertiesFile(patchExistingPropertiesFile, keyBase + ".date", 
7875           GrouperInstallerUtils.dateMinutesSecondsFormat.format(new Date()), true);
7876 
7877       //if we arent installing the patch
7878       if (!installPatch) {
7879         
7880         boolean temporary = false;
7881         
7882         //if installing up to a patch level, and not specifying about next time, make it temporary
7883         if (this.installPatchesUpToACertainPatchLevel && GrouperInstallerUtils.isBlank(GrouperInstallerUtils.propertiesValue("grouperInstaller.autorun.promptAboutPatchNextTime", false))) {
7884           temporary = true;
7885         } else if (this.installCertainSpecifiedPatches && GrouperInstallerUtils.isBlank(GrouperInstallerUtils.propertiesValue("grouperInstaller.autorun.promptAboutPatchNextTime", false))) {
7886           temporary = true;
7887 
7888         } else {
7889           System.out.println("Would you like to be prompted about this patch next time? (t|f)? [t]: ");
7890 
7891           temporary = readFromStdInBoolean(true, "grouperInstaller.autorun.promptAboutPatchNextTime");
7892         }
7893 
7894         GrouperInstallerPatchStatus grouperInstallerPatchStatus = null;
7895 
7896         if (temporary) {
7897           grouperInstallerPatchStatus = GrouperInstallerPatchStatus.skippedTemporarily;
7898         } else {
7899           grouperInstallerPatchStatus = GrouperInstallerPatchStatus.skippedPermanently;
7900         }
7901 
7902         editPropertiesFile(patchExistingPropertiesFile, keyBase + ".state", 
7903             grouperInstallerPatchStatus.name(), true);
7904         System.out.println("");
7905         continue OUTER;
7906       }
7907 
7908       if (requiresRestart && !this.grouperStopped) {
7909         System.out.println("This patch requires all processes that user Grouper to be stopped.\n  "
7910             + "Please stop these processes if they are running and press <enter> to continue...");
7911         this.grouperStopped = true;
7912         readFromStdIn("grouperInstaller.autorun.continueAfterPatchStopProcesses");
7913       }
7914       
7915       Map<String, String> patchDirToApplicationPath = new LinkedHashMap<String, String>();
7916       patchDirToApplicationPath.put("files", this.upgradeExistingApplicationDirectoryString);
7917       patchDirToApplicationPath.put("classes", this.upgradeExistingClassesDirectoryString);
7918       patchDirToApplicationPath.put("lib", this.upgradeExistingLibDirectoryString);
7919       patchDirToApplicationPath.put("bin", this.upgradeExistingBinDirectoryString);
7920 
7921       boolean patchHasProblem = false;
7922       
7923       //we are installing this patch, lets see if the files are there...
7924       //this.upgradeExistingApplicationDirectoryString
7925       //patchUntarredDir
7926       File oldDir = new File(patchUntarredDir.getAbsolutePath() + File.separator + "old");
7927       File newDir = new File(patchUntarredDir.getAbsolutePath() + File.separator + "new");
7928       {
7929 
7930         for (String patchDir : patchDirToApplicationPath.keySet()) {
7931           
7932           String applicationPath = patchDirToApplicationPath.get(patchDir);
7933 
7934           File oldDirFiles = new File(oldDir.getAbsoluteFile() + File.separator + patchDir);
7935           File newDirFiles = new File(newDir.getAbsoluteFile() + File.separator + patchDir);
7936           
7937           if (oldDirFiles.exists() && oldDirFiles.isDirectory()) {
7938           
7939             // relative, e.g. WEB-INF/jsp/someFile.jsp
7940             Set<String> oldFileRelativePaths = GrouperInstallerUtils.fileDescendantRelativePaths(oldDirFiles);
7941             
7942             for (String oldFilePath : GrouperInstallerUtils.nonNull(oldFileRelativePaths)) {
7943               File oldFileInPatch = new File(oldDirFiles.getAbsolutePath() + File.separator + oldFilePath);
7944               File newFileInPatch = new File(newDirFiles.getAbsolutePath() + File.separator + oldFilePath);
7945 
7946               oldFilePath = patchFixFilePath(applicationPath, patchDir, oldFilePath, originalAppToUpgrade);
7947 
7948               File oldFileInGrouper = new File(applicationPath + oldFilePath);
7949   
7950               oldFileInGrouper = fixLibraryFileIfFoundAndDifferent(oldFileInGrouper, originalAppToUpgrade);
7951 
7952               if (!oldFileInPatch.exists() || !oldFileInPatch.isFile()) {
7953                 throw new RuntimeException("Why does file not exist or not file??? " + oldFileInPatch.getAbsolutePath());
7954               }
7955               boolean deletedNewPatchFile = !newFileInPatch.exists();
7956               boolean deletedGrouperFile = !oldFileInGrouper.exists();
7957               //if both deleted thats ok
7958               if ((!deletedGrouperFile || !deletedNewPatchFile) &&
7959                  ( !oldFileInGrouper.exists() || !oldFileInGrouper.isFile() 
7960                   || (!GrouperInstallerUtils.contentEquals(oldFileInPatch, oldFileInGrouper)
7961                       //patch is already applied?  thats ok i guess
7962                       && !GrouperInstallerUtils.contentEquals(newFileInPatch, oldFileInGrouper)))) {
7963                 
7964                 System.out.println("Problem applying patch since this patch old file:\n  " + oldFileInPatch.getAbsolutePath() 
7965                     + "\n  is not the same as what the patch expects:\n  " + oldFileInGrouper.getAbsolutePath()
7966                     + "\n  Do you want to force install this patch (t|f)? [f]: ");
7967                 
7968                 boolean forceInstallPatch = readFromStdInBoolean(false, "grouperInstaller.autorun.forceInstallPatch");
7969                 
7970                 if (!forceInstallPatch) {
7971                   System.out.println("Cannot apply patch since this patch file:\n  " + oldFileInPatch.getAbsolutePath() 
7972                       + "\n  is not the same as what the patch expects:\n  " + oldFileInGrouper.getAbsolutePath());
7973                   patchHasProblem = true;
7974                 }
7975               }
7976             }
7977           }
7978         }
7979       }
7980 
7981       //lets make sure that files which are new which dont have an old version do not exist in the application
7982       for (String patchDir : patchDirToApplicationPath.keySet()) {
7983         
7984         String applicationPath = patchDirToApplicationPath.get(patchDir);
7985 
7986         File newDirFiles = new File(newDir.getAbsoluteFile() + File.separator + patchDir);
7987         File oldDirFiles = new File(oldDir.getAbsoluteFile() + File.separator + patchDir);
7988         
7989         if (newDirFiles.exists() && newDirFiles.isDirectory()) {
7990         
7991           // relative, e.g. WEB-INF/jsp/someFile.jsp
7992           Set<String> newFileRelativePaths = GrouperInstallerUtils.fileDescendantRelativePaths(newDirFiles);
7993 
7994           Set<String> oldFileRelativePaths = (oldDirFiles.exists() && oldDirFiles.isDirectory()) ? 
7995               GrouperInstallerUtils.fileDescendantRelativePaths(oldDirFiles) : new HashSet<String>();
7996 
7997           for (String newFilePath : GrouperInstallerUtils.nonNull(newFileRelativePaths)) {
7998 
7999             File newFileInPatch = new File(newDirFiles.getAbsoluteFile() + File.separator + newFilePath);
8000             
8001             newFilePath = patchFixFilePath(applicationPath, patchDir, newFilePath, originalAppToUpgrade);
8002 
8003             File oldFileInGrouper = new File(applicationPath + newFilePath);
8004 
8005             oldFileInGrouper = fixLibraryFileIfFoundAndDifferent(oldFileInGrouper, originalAppToUpgrade);
8006 
8007             if (!newFileInPatch.isFile()) {
8008               continue;
8009             }
8010             
8011             //if there wasnt a corresponding old file path
8012             if (!oldFileRelativePaths.contains(newFilePath) && !GrouperInstallerUtils.contentEquals(oldFileInGrouper, newFileInPatch)) {
8013 
8014               //then the file shouldnt exist
8015               if (oldFileInGrouper.exists()) {
8016 
8017                 System.out.println("Problem applying patch since this file:\n  " + oldFileInGrouper.getAbsolutePath() 
8018                   + "\n  should not exist yet\n  Do you want to force install this patch (t|f)? [f]: ");
8019             
8020                 boolean forceInstallPatch = readFromStdInBoolean(false, "grouperInstaller.autorun.forceInstallPatch");
8021                 
8022                 if (!forceInstallPatch) {
8023                 
8024                 
8025                   System.out.println("Cannot apply patch since this patch file:\n  " + newFileInPatch.getAbsolutePath() 
8026                       + "\n  is supposed to be new, but it already exists:\n  " + oldFileInGrouper.getAbsolutePath());
8027                   patchHasProblem = true;
8028 
8029                 }
8030               }
8031             }
8032           }
8033         }
8034       }
8035 
8036       if (patchHasProblem) {
8037         editPropertiesFile(patchExistingPropertiesFile, keyBase + ".state", 
8038             GrouperInstallerPatchStatus.error.name(), true);
8039 
8040         continue OUTER;
8041       }
8042 
8043       //so far so good, all the old files are all good, install the patch
8044       for (String patchDir : patchDirToApplicationPath.keySet()) {
8045         
8046         String applicationPath = patchDirToApplicationPath.get(patchDir);
8047 
8048         File newDirFiles = new File(newDir.getAbsoluteFile() + File.separator + patchDir);
8049         
8050         if (newDirFiles.exists() && newDirFiles.isDirectory()) {
8051         
8052           // relative, e.g. WEB-INF/jsp/someFile.jsp
8053           Set<String> newFileRelativePaths = GrouperInstallerUtils.fileDescendantRelativePaths(newDirFiles);
8054           
8055           for (String newFilePath : GrouperInstallerUtils.nonNull(newFileRelativePaths)) {
8056             // adjust for jars in web apps
8057             //patchDir (e.g. lib), newFilePath (e.g. something.jar or grouper/something.jar), originalAppToUpgrade (e.g. UI)
8058             File newFileInPatch = new File(newDirFiles.getAbsolutePath() + File.separator + newFilePath);
8059             if (!newFileInPatch.isFile()) {
8060               continue;
8061             }
8062             newFilePath = patchFixFilePath(applicationPath, patchDir, newFilePath, originalAppToUpgrade);
8063             File oldFileInGrouper = new File(applicationPath + newFilePath);
8064             oldFileInGrouper = fixLibraryFileIfFoundAndDifferent(oldFileInGrouper, originalAppToUpgrade);
8065 
8066             if (!oldFileInGrouper.exists() && !oldFileInGrouper.getParentFile().exists()) {
8067               GrouperInstallerUtils.mkdirs(oldFileInGrouper.getParentFile());
8068             }
8069             System.out.println("Applying file: " + oldFileInGrouper.getAbsolutePath());
8070             GrouperInstallerUtils.copyFile(newFileInPatch, oldFileInGrouper, false);
8071           }
8072         }
8073         
8074         File oldDirFiles = new File(oldDir.getAbsoluteFile() + File.separator + patchDir);
8075         
8076         if (oldDirFiles.exists() && oldDirFiles.isDirectory()) {
8077         
8078           // relative, e.g. WEB-INF/jsp/someFile.jsp
8079           Set<String> oldFileRelativePaths = GrouperInstallerUtils.fileDescendantRelativePaths(oldDirFiles);
8080           
8081           for (String oldFilePath : GrouperInstallerUtils.nonNull(oldFileRelativePaths)) {
8082             File oldFileInPatch = new File(oldDirFiles.getAbsolutePath() + File.separator + oldFilePath);
8083             File newFileInPatch = new File(newDirFiles.getAbsolutePath() + File.separator + oldFilePath);
8084             File oldFileInGrouper = new File(applicationPath + oldFilePath);
8085             oldFileInGrouper = fixLibraryFileIfFoundAndDifferent(oldFileInGrouper, originalAppToUpgrade);
8086 
8087             if (oldFileInPatch.exists() && !newFileInPatch.exists() && oldFileInGrouper.exists() && oldFileInGrouper.isFile()) {
8088 
8089               System.out.println("Deleting file: " + oldFileInGrouper.getAbsolutePath());
8090               GrouperInstallerUtils.fileDelete(oldFileInGrouper);
8091               
8092             }
8093           }
8094         }
8095       }
8096       
8097       //fixLibDir(patchDirToApplicationPath.get("lib"), originalAppToUpgrade);
8098 
8099       this.patchesInstalled.add(keyBase);
8100       System.out.println("Patch successfully applied: " + keyBase);
8101       
8102       editPropertiesFile(patchExistingPropertiesFile, keyBase + ".state", 
8103           GrouperInstallerPatchStatus.applied.name(), true);
8104       System.out.println("");
8105     }
8106 
8107     if (!foundNewPatch) {
8108       System.out.println("There are no new " + thisAppToUpgrade + " patches to install\n");
8109       return false;
8110     } 
8111     return true;
8112   }
8113 
8114   /**
8115    * <pre>
8116    * patch file extra grouper prefix pattern
8117    * ^grouper[/\\][^/\\]+[/\\]([^/\\]+)$
8118    * starts with grouper, then a slash, then capture a dir and jar filename
8119    * </pre>
8120    */
8121   private static Pattern patchFileExtraGrouperPrefixPattern = Pattern.compile("^grouper[/\\\\]([^/\\\\]+[/\\\\][^/\\\\]+)$");
8122   
8123   /**
8124    * applicationPath (e.g. lib), newFilePath (e.g. something.jar or grouper/something.jar), originalAppToUpgrade (e.g. UI)
8125    * @param applicationPath
8126    * @param patchDir
8127    * @param newFilePath
8128    * @param originalAppToUpgrade
8129    * @return String of new newFilePath
8130    */
8131   public String patchFixFilePath(String applicationPath, String patchDir, String newFilePath, AppToUpgrade originalAppToUpgrade) {
8132 
8133     if ("lib".equals(patchDir)) {
8134       // if this is the api then there should be something between lib and the jar
8135       String jarName = newFilePath;
8136       {
8137 //        Matcher matcher = patchFileExtraGrouperPrefixPattern.matcher(newFilePath);
8138 //        if (matcher.matches()) {
8139 //          jarName = matcher.group(1);
8140 //        }
8141         jarName = GrouperInstallerUtils.suffixAfterChar(newFilePath.replace("\\", "/"), '/');
8142 
8143       }
8144       
8145 
8146       if (originalAppToUpgrade.isApiOrganized()) {
8147         
8148         String noSlashApplicationPath = GrouperInstallerUtils.stripLastSlashIfExists(applicationPath);
8149         // if the application path has "grouper" already, then just put jarname on it
8150         if (!noSlashApplicationPath.endsWith("lib")) {
8151           newFilePath = jarName;
8152         } else {
8153           //if application is just lib, then make sure there is something in front of the jarname
8154           if (GrouperInstallerUtils.equals(newFilePath, jarName)) {
8155             newFilePath = "grouper/" + jarName;
8156           }
8157         }
8158          
8159       } else {
8160         // if this is a UI or WS, then we only want the jar
8161         newFilePath = jarName;
8162       }
8163     }
8164     return newFilePath;
8165   }
8166 
8167   
8168   /**
8169    * @param keyBase
8170    * @return if should revert patch
8171    */
8172   private boolean shouldRevertCertainSpecifiedPatches(String keyBase) {
8173     List<String> revertUpToThesePatchLevelsList = GrouperInstallerUtils.splitTrimToList(this.revertCertainSpecifiedPatchesList, ",");
8174     return revertUpToThesePatchLevelsList.contains(keyBase);
8175   }
8176 
8177   /**
8178    * @param keyBase
8179    * @return if should install patch
8180    */
8181   private boolean shouldInstallCertainSpecifiedPatches(String keyBase) {
8182     
8183     List<String> installUpToThesePatchLevelsList = GrouperInstallerUtils.splitTrimToList(this.installCertainSpecifiedPatchesList, ",");
8184     return installUpToThesePatchLevelsList.contains(keyBase);
8185   }
8186   
8187   /**
8188    * @param keyBase
8189    * @return if should install patch
8190    */
8191   private boolean shouldInstallPatchUpToLevel(String keyBase) {
8192     boolean installPatch = false;
8193 
8194     //e.g. ^grouper_v(\\d+)_(\\d+)_(\\d+)_(api|ws|ui|psp|pspng)_patch_(\\d+)$
8195     Matcher patchNameMatcher = patchNamePattern.matcher(keyBase);
8196     if (!patchNameMatcher.matches()) {
8197       throw new RuntimeException("Invalid patch name: " + keyBase);
8198     }
8199     
8200     String grouperVersionInstallPatch = patchNameMatcher.group(1) + "." + patchNameMatcher.group(2) + "." + patchNameMatcher.group(3);
8201     String systemInstallPatch = patchNameMatcher.group(4);
8202     int numberInstallPatch = GrouperInstallerUtils.intValue(patchNameMatcher.group(5));
8203 
8204     
8205     String[] installUpToThesePatchLevels = GrouperInstallerUtils.splitTrim(this.installPatchesUpToThesePatchLevels, ",");
8206     for (String patchName : installUpToThesePatchLevels) {
8207 
8208       //e.g. ^grouper_v(\\d+)_(\\d+)_(\\d+)_(api|ws|ui|psp|pspng)_patch_(\\d+)$
8209       patchNameMatcher = patchNamePattern.matcher(patchName);
8210       if (!patchNameMatcher.matches()) {
8211         throw new RuntimeException("Invalid patch name: " + patchName);
8212       }
8213       
8214       String grouperVersionUpToPatch = patchNameMatcher.group(1) + "." + patchNameMatcher.group(2) + "." + patchNameMatcher.group(3);
8215       String systemUpToPatch = patchNameMatcher.group(4);
8216       int numberUpToPatch = GrouperInstallerUtils.intValue(patchNameMatcher.group(5));
8217 
8218       if (GrouperInstallerUtils.equals(systemInstallPatch, systemUpToPatch)
8219           && GrouperInstallerUtils.equals(grouperVersionInstallPatch, grouperVersionUpToPatch)
8220           && numberInstallPatch <= numberUpToPatch) {
8221         installPatch = true;
8222         break;
8223       }
8224       
8225     }
8226     return installPatch;
8227   }
8228   
8229   /**
8230    * fix the index file
8231    * @param thisAppToUpgrade app to upgrade to check
8232    */
8233   private void fixIndexFile(AppToUpgrade thisAppToUpgrade) {
8234 
8235     if (thisAppToUpgrade == AppToUpgrade.CLIENT) {
8236       throw new RuntimeException("Cant fix index file for " + thisAppToUpgrade);
8237     }
8238     
8239     Properties patchesExistingProperties = patchExistingProperties();
8240 
8241     String grouperVersion = this.grouperVersionOfJar().toString();
8242 
8243     grouperVersion = GrouperInstallerUtils.replace(grouperVersion, ".", "_");
8244 
8245     //lets download all patches
8246     int nextPatchIndex = downloadPatches(thisAppToUpgrade, grouperVersion);
8247     
8248     File patchExistingPropertiesFile = patchExistingPropertiesFile();
8249 
8250     Map<String, String> patchDirToApplicationPath = new LinkedHashMap<String, String>();
8251     patchDirToApplicationPath.put("files", this.upgradeExistingApplicationDirectoryString);
8252     patchDirToApplicationPath.put("classes", this.upgradeExistingClassesDirectoryString);
8253     patchDirToApplicationPath.put("lib", this.upgradeExistingLibDirectoryString);
8254     patchDirToApplicationPath.put("bin", this.upgradeExistingBinDirectoryString);
8255 
8256     //map of full patch file name to the patch number that is installed
8257     Map<String, Integer> fileInMoreRecentPatchMap = new HashMap<String, Integer>();
8258 
8259     boolean patchesOverallOk = true;
8260     
8261     //process patches from greatest to least
8262     for (int i=nextPatchIndex-1;i>=0;i--) {
8263       
8264       //grouper_v2_2_1_api_patch_0.state
8265       String patchName = "grouper_v" + grouperVersion + "_" + thisAppToUpgrade.name().toLowerCase() + "_patch_" + i;
8266 
8267       String key = patchName + ".state";
8268 
8269       //see what is already there
8270       String existingState = patchesExistingProperties.getProperty(key);
8271       
8272       GrouperInstallerPatchStatus grouperInstallerPatchStatus = GrouperInstallerPatchStatus.valueOfIgnoreCase(existingState, false, true);
8273 
8274       File patchUntarredDir = new File(this.grouperTarballDirectoryString + "patches" + File.separator + patchName);
8275 
8276       //keep track that we skipped this in the patch properties file
8277       //editPropertiesFile(patchExistingPropertiesFile, keyBase + ".date", 
8278       //    GrouperInstallerUtils.dateMinutesSecondsFormat.format(new Date()));
8279       //editPropertiesFile(patchExistingPropertiesFile, keyBase + ".state", 
8280       //    grouperInstallerPatchStatus.name());
8281 
8282       boolean patchHasProblem = false;
8283       boolean patchHasAtLeastOneFile = false;
8284       boolean patchHasAtLeastOneFileInAnotherPatch = false;
8285       Set<String> patchErrors = new LinkedHashSet<String>();
8286       
8287       //keep track of patch paths (full path in patch)
8288       Set<String> patchPaths = new HashSet<String>();
8289       
8290       //we are installing this patch, lets see if the files are there...
8291       //this.upgradeExistingApplicationDirectoryString
8292       //patchUntarredDir
8293       File newDir = new File(patchUntarredDir.getAbsolutePath() + File.separator + "new");
8294       //loop through lib, classes, files
8295       for (String patchDir : patchDirToApplicationPath.keySet()) {
8296 
8297         String applicationPath = patchDirToApplicationPath.get(patchDir);
8298 
8299         File newDirFiles = new File(newDir.getAbsoluteFile() + File.separator + patchDir);
8300 
8301         // relative, e.g. WEB-INF/jsp/someFile.jsp
8302         Set<String> newFileRelativePaths = GrouperInstallerUtils.fileDescendantRelativePaths(newDirFiles);
8303         // go through all files of the patches in the new dir
8304         for (String newFilePath : GrouperInstallerUtils.nonNull(newFileRelativePaths)) {
8305 
8306           String patchPath = patchDir + File.separator + newFilePath;
8307           
8308           Integer existsInPatchVersion = fileInMoreRecentPatchMap.get(patchPath);
8309           
8310           //if this file was in a newer patch, then thats ok
8311           if (existsInPatchVersion != null) {
8312             //this file is ok, its in a more recent patch
8313             patchHasAtLeastOneFileInAnotherPatch = true;
8314             continue;
8315           }
8316 
8317           File newFileInGrouper = new File(applicationPath + newFilePath);
8318 
8319           File newFileInPatch = new File(newDirFiles.getAbsolutePath() + File.separator + newFilePath);
8320           
8321           //see if the contents of the patch match those in grouper
8322           if (!GrouperInstallerUtils.contentEquals(newFileInPatch, newFileInGrouper)) {
8323 
8324             patchErrors.add("Problem in patch:\n  " + newFileInPatch.getAbsolutePath() 
8325                 + "\n  is not the same as what the patch expects:\n  " + newFileInGrouper.getAbsolutePath());
8326             patchHasProblem = true;
8327           } else {
8328 
8329             patchPaths.add(patchPath);
8330 
8331             patchHasAtLeastOneFile = true;
8332           }
8333         }
8334       }
8335         
8336       //is any file installed?  or if there are only files in other patches... hmm
8337       if (patchHasAtLeastOneFile || (patchHasAtLeastOneFileInAnotherPatch && !patchHasProblem )) {
8338         
8339         //add files in this patch to the list
8340         for (String patchPath : patchPaths) {
8341           fileInMoreRecentPatchMap.put(patchPath, i);
8342         }
8343         
8344         //one or more of the files in the patch had a problem
8345         if (patchHasProblem) {
8346           for (String patchError: patchErrors) {
8347             System.out.println(patchError);
8348           }
8349           if (grouperInstallerPatchStatus == null || (grouperInstallerPatchStatus != GrouperInstallerPatchStatus.applied
8350               && grouperInstallerPatchStatus != GrouperInstallerPatchStatus.error)) {
8351             patchesOverallOk = false;
8352             editPropertiesFile(patchExistingPropertiesFile, patchName + ".date", 
8353                 GrouperInstallerUtils.dateMinutesSecondsFormat.format(new Date()), true);
8354             editPropertiesFile(patchExistingPropertiesFile, patchName + ".state", 
8355                 GrouperInstallerPatchStatus.applied.name(), true);
8356             System.out.println("Patch " + patchName + " was listed as " + grouperInstallerPatchStatus + " but was changed to applied (even though there are files missing)");
8357             
8358           }
8359           continue;          
8360         }
8361         
8362         if (grouperInstallerPatchStatus == null || grouperInstallerPatchStatus != GrouperInstallerPatchStatus.applied) {
8363           patchesOverallOk = false;
8364           editPropertiesFile(patchExistingPropertiesFile, patchName + ".date", 
8365               GrouperInstallerUtils.dateMinutesSecondsFormat.format(new Date()), true);
8366           editPropertiesFile(patchExistingPropertiesFile, patchName + ".state", 
8367               GrouperInstallerPatchStatus.applied.name(), true);
8368           System.out.println("Patch " + patchName + " was listed as " + grouperInstallerPatchStatus + " but was changed to applied");
8369           
8370         }
8371         
8372       } else {
8373         if (grouperInstallerPatchStatus == null || grouperInstallerPatchStatus != GrouperInstallerPatchStatus.applied) {
8374           continue;
8375         }
8376         
8377         patchesOverallOk = false;
8378         editPropertiesFile(patchExistingPropertiesFile, patchName + ".date", 
8379             GrouperInstallerUtils.dateMinutesSecondsFormat.format(new Date()), true);
8380         editPropertiesFile(patchExistingPropertiesFile, patchName + ".state", 
8381             GrouperInstallerPatchStatus.skippedTemporarily.name(), true);
8382         System.out.println("Patch " + patchName + " was listed as applied but was changed to skippedTemporarily");
8383         continue;
8384       }
8385 
8386     }
8387 
8388     //tell the properties file that we have fixed the index file now
8389     editPropertiesFile(patchExistingPropertiesFile, "grouperInstallerLastFixedIndexFile.date", 
8390         GrouperInstallerUtils.dateMinutesSecondsFormat.format(new Date()), true);
8391   
8392     if (patchesOverallOk) {
8393       System.out.println("Patches for " + thisAppToUpgrade + " for version " + grouperVersion + " were in the index file correctly");
8394     }
8395   }
8396   
8397   /**
8398    * get all patches
8399    * @param thisAppToUpgrade app to upgrade to check
8400    * @param grouperVersion
8401    * @return next patch index
8402    */
8403   private int downloadPatches(AppToUpgrade thisAppToUpgrade, String grouperVersion) {
8404 
8405     if (thisAppToUpgrade == AppToUpgrade.CLIENT) {
8406       throw new RuntimeException("Cant install patches for " + thisAppToUpgrade);
8407     }
8408     
8409     Map<Integer, String> patchNumberToNameBase = new LinkedHashMap<Integer, String>();
8410     
8411     int nextPatchIndex = 0;
8412 
8413     OUTER: for (int i=0;i<1000;i++) {
8414 
8415       //grouper_v2_2_1_api_patch_0.state
8416       String keyBase = "grouper_v" + grouperVersion + "_" + thisAppToUpgrade.name().toLowerCase() + "_patch_" + i;
8417 
8418       patchNumberToNameBase.put(i, keyBase);
8419 
8420       //lets see if it exists on the server
8421       File patchUntarredDir = downloadAndUnzipPatch(keyBase);
8422 
8423       //if no more patches
8424       if (patchUntarredDir == null) {
8425         System.out.println("");
8426         break OUTER;
8427       }
8428 
8429       nextPatchIndex = i+1;
8430     }
8431 
8432     return nextPatchIndex;
8433 
8434   }
8435   
8436   /**
8437    * 
8438    * @param patchName e.g. grouper_v2_2_1_api_patch_0.tar.gz
8439    * @return the directory of the unzipped patch
8440    */
8441   public File downloadAndUnzipPatch(String patchName) {
8442     String urlToDownload = GrouperInstallerUtils.propertiesValue("download.server.url", true);
8443     
8444     if (!urlToDownload.endsWith("/")) {
8445       urlToDownload += "/";
8446     }
8447     urlToDownload += "release/";
8448     
8449     //e.g. 2.2.2
8450     Matcher patchNameMatcher = patchNamePattern.matcher(patchName);
8451     if (!patchNameMatcher.matches()) {
8452       throw new RuntimeException("Invalid patch name: " + patchName);
8453     }
8454     
8455     //String grouperVersion = GrouperInstallerUtils.propertiesValue("grouper.version", true);
8456     String grouperVersion = patchNameMatcher.group(1) + "." + patchNameMatcher.group(2) + "." + patchNameMatcher.group(3);
8457     
8458     urlToDownload +=  grouperVersion + "/patches/" + patchName + ".tar.gz";
8459 
8460     File patchFile = new File(this.grouperTarballDirectoryString + "patches" + File.separator + patchName + ".tar.gz");
8461     
8462     if (GrouperInstallerUtils.propertiesValueBoolean("grouperInstaller.default.downloadPatches", true, false)) {
8463 
8464       boolean foundFile = downloadFile(urlToDownload, patchFile.getAbsolutePath(), true, "Patch doesnt exist yet (not an error): ", 
8465           "grouperInstaller.autorun.useLocalPatchIfExists");
8466       
8467       if (!foundFile) {
8468 
8469         //if we are doing test patches
8470         if (GrouperInstallerUtils.propertiesValueBoolean("grouperInstaller.useTestPatches", false, false)) {
8471           String testUrlToDownload = GrouperInstallerUtils.replace(urlToDownload, ".tar.gz", "_test.tar.gz");
8472           
8473           //its a test url, but download to the same file name
8474           foundFile = downloadFile(testUrlToDownload, patchFile.getAbsolutePath(), true, "Patch doesnt exist yet (not an error): ", 
8475               "grouperInstaller.autorun.useLocalPatchIfExists");
8476         }
8477 
8478         if (!foundFile) {
8479           return null;
8480         }
8481       }
8482     } else {
8483       if (!patchFile.exists()) {
8484         return null;
8485       }
8486     }
8487     
8488     //####################################
8489     //unzip/untar the patch file
8490     
8491     File unzippedFile = unzip(patchFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalPatchIfExists");
8492     File untarredDir = untar(unzippedFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalPatchIfExists", null);
8493     return untarredDir;
8494   }
8495 
8496   /**
8497    * 
8498    * @param branchName
8499    * @return the directory of the unzipped source repo
8500    */
8501   public File downloadAndUnzipGrouperSource(String branchName) {
8502     String urlToDownload = GrouperInstallerUtils.propertiesValue("download.source.url", false);
8503     
8504     if (GrouperInstallerUtils.isBlank(urlToDownload)) {
8505       urlToDownload = "https://github.com/Internet2/grouper/archive/$BRANCH_NAME$.zip";
8506     }
8507     
8508     urlToDownload = GrouperInstallerUtils.replace(urlToDownload, "$BRANCH_NAME$", branchName);
8509 
8510     String fileToDownload = GrouperInstallerUtils.substringAfterLast(urlToDownload, "/");
8511     
8512     File sourceFile = new File(this.grouperTarballDirectoryString + fileToDownload);
8513     
8514     if (GrouperInstallerUtils.propertiesValueBoolean("grouperInstaller.default.downloadSource", true, false)) {
8515 
8516       downloadFile(urlToDownload, sourceFile.getAbsolutePath(), "grouperInstaller.autorun.createPatchDownloadSourceUseLocalIfExist");
8517       
8518     } else {
8519       if (!sourceFile.exists()) {
8520         throw new RuntimeException("Cant find grouper source");
8521       }
8522     }
8523     
8524     //####################################
8525     //unzip/untar the source file
8526     File unzippedDir = unzipFromZip(sourceFile.getAbsolutePath(), "grouperInstaller.autorun.createPatchDownloadSourceUseLocalIfExist");
8527     return unzippedDir;
8528   }
8529 
8530   /**
8531    * 
8532    * @param branchName
8533    * @return the directory of the unzipped source repo
8534    */
8535   public File downloadAndUnzipPspSource(String branchName) {
8536     String urlToDownload = GrouperInstallerUtils.propertiesValue("download.pspSource.url", false);
8537     
8538     if (GrouperInstallerUtils.isBlank(urlToDownload)) {
8539       urlToDownload = "https://github.com/Internet2/grouper-psp/archive/$BRANCH_NAME$.zip";
8540     }
8541     
8542     urlToDownload = GrouperInstallerUtils.replace(urlToDownload, "$BRANCH_NAME$", branchName);
8543 
8544     String fileToDownload = GrouperInstallerUtils.substringAfterLast(urlToDownload, "/");
8545     
8546     File sourceFile = new File(this.grouperTarballDirectoryString + fileToDownload);
8547     
8548     if (GrouperInstallerUtils.propertiesValueBoolean("grouperInstaller.default.downloadSource", true, false)) {
8549 
8550       downloadFile(urlToDownload, sourceFile.getAbsolutePath(), "grouperInstaller.autorun.createPatchDownloadSourceUseLocalIfExist");
8551       
8552     } else {
8553       if (!sourceFile.exists()) {
8554         throw new RuntimeException("Cant find grouper psp source");
8555       }
8556     }
8557     
8558     //####################################
8559     //unzip/untar the source file
8560     File unzippedDir = unzipFromZip(sourceFile.getAbsolutePath(), "grouperInstaller.autorun.createPatchDownloadSourceUseLocalIfExist");
8561     return unzippedDir;
8562   }
8563 
8564   /**
8565    * 
8566    */
8567   public static enum GrouperInstallerPatchStatus {
8568 
8569     /**
8570      * patch was applied
8571      */
8572     applied, 
8573     
8574     /**
8575      * patch was removed
8576      */
8577     reverted, 
8578     
8579     /**
8580      * patch was skipped temporarily, prompt again
8581      */
8582     skippedTemporarily, 
8583 
8584     /**
8585      * patch had an error applying
8586      */
8587     error, 
8588 
8589     /**
8590      * patch was skipped permanently, dont prompt again
8591      */
8592     skippedPermanently;
8593 
8594     /**
8595      * 
8596      * @param string
8597      * @param exceptionIfNotFound
8598      * @param exceptionIfInvalid
8599      * @return the patch status
8600      */
8601     public static GrouperInstallerPatchStatus valueOfIgnoreCase(String string, boolean exceptionIfNotFound, boolean exceptionIfInvalid) {
8602       return GrouperInstallerUtils.enumValueOfIgnoreCase(GrouperInstallerPatchStatus.class, string, exceptionIfNotFound, exceptionIfInvalid);
8603     }
8604     
8605   }
8606 
8607   /**
8608    * patch status api
8609    */
8610   private void patchStatusApi() {
8611     this.patchStatus(AppToUpgrade.API);
8612   }
8613 
8614 
8615   /**
8616    * patch the api
8617    */
8618   private void patchApi() {
8619     this.downloadAndInstallPatches(AppToUpgrade.API, AppToUpgrade.API);
8620   }
8621 
8622   /**
8623    * fix index file api
8624    */
8625   private void fixIndexFileApi() {
8626     this.fixIndexFile(AppToUpgrade.API);
8627   }
8628 
8629   /**
8630    * fix index file ui
8631    */
8632   private void fixIndexFileUi() {
8633     this.fixIndexFile(AppToUpgrade.UI);
8634     this.fixIndexFile(AppToUpgrade.API);
8635   }
8636 
8637   /**
8638    * fix index file ws
8639    */
8640   private void fixIndexFileWs() {
8641     this.fixIndexFile(AppToUpgrade.WS);
8642     this.fixIndexFile(AppToUpgrade.API);
8643   }
8644 
8645   /**
8646    * fix index file psp
8647    */
8648   private void fixIndexFilePsp() {
8649     this.fixIndexFile(AppToUpgrade.PSP);
8650     this.fixIndexFile(AppToUpgrade.API);
8651   }
8652 
8653   /**
8654    * fix index file psp
8655    */
8656   private void fixIndexFilePspng() {
8657     this.fixIndexFile(AppToUpgrade.PSPNG);
8658     this.fixIndexFile(AppToUpgrade.API);
8659   }
8660 
8661   /**
8662    * patch status ui
8663    */
8664   private void patchStatusUi() {
8665     this.patchStatus(AppToUpgrade.API);
8666     this.patchStatus(AppToUpgrade.UI);
8667   }
8668 
8669   /**
8670    * patch status ws
8671    */
8672   private void patchStatusWs() {
8673     this.patchStatus(AppToUpgrade.API);
8674     this.patchStatus(AppToUpgrade.WS);
8675   }
8676 
8677   /**
8678    * patch status psp
8679    */
8680   private void patchStatusPsp() {
8681     this.patchStatus(AppToUpgrade.API);
8682     this.patchStatus(AppToUpgrade.PSP);
8683   }
8684 
8685   /**
8686    * patch status pspng
8687    */
8688   private void patchStatusPspng() {
8689     this.patchStatus(AppToUpgrade.API);
8690     this.patchStatus(AppToUpgrade.PSPNG);
8691   }
8692 
8693 
8694   /**
8695    * patch the client
8696    */
8697   private void patchUi() {
8698     this.downloadAndInstallPatches(AppToUpgrade.API, AppToUpgrade.UI);
8699     boolean patchesApplied = this.downloadAndInstallPatches(AppToUpgrade.UI, AppToUpgrade.UI);
8700     if (patchesApplied && (this.grouperInstallerMainFunction == GrouperInstallerMainFunction.patch 
8701         || this.grouperInstallerMainFunction == GrouperInstallerMainFunction.upgrade)) {
8702       System.out.print("Since patches were applied, you should delete files in your app server work directory,"
8703           + "\n  in tomcat it is named 'work'.  Hit <enter> to continue: ");
8704       readFromStdIn("grouperInstaller.autorun.continueAfterDeleteUiWorkDirectory");
8705     }
8706   }
8707   
8708   /**
8709    * patch the client
8710    */
8711   private void patchWs() {
8712     this.downloadAndInstallPatches(AppToUpgrade.API, AppToUpgrade.WS);
8713     boolean patchesApplied = this.downloadAndInstallPatches(AppToUpgrade.WS, AppToUpgrade.WS);
8714     if (patchesApplied && (this.grouperInstallerMainFunction == GrouperInstallerMainFunction.patch 
8715         || this.grouperInstallerMainFunction == GrouperInstallerMainFunction.upgrade)) {
8716       System.out.print("Since patches were applied, you should delete files in your app server work directory,"
8717           + "\n  in tomcat it is named 'work'.  Hit <enter> to continue: ");
8718       readFromStdIn("grouperInstaller.autorun.continueAfterDeleteWsWorkDirectory");
8719     }
8720   }
8721   
8722   /**
8723    * patch the psp
8724    */
8725   private void patchPsp() {
8726     this.downloadAndInstallPatches(AppToUpgrade.API, AppToUpgrade.PSP);
8727     this.downloadAndInstallPatches(AppToUpgrade.PSP, AppToUpgrade.PSP);
8728   }
8729 
8730   /**
8731    * patch the pspng
8732    */
8733   private void patchPspng() {
8734     this.downloadAndInstallPatches(AppToUpgrade.API, AppToUpgrade.PSPNG);
8735     this.downloadAndInstallPatches(AppToUpgrade.PSPNG, AppToUpgrade.PSPNG);
8736   }
8737 
8738   /**
8739    * revert patch the client
8740    */
8741   private void patchRevertApi() {
8742     this.revertPatches(AppToUpgrade.API, AppToUpgrade.API);
8743   }
8744 
8745   /**
8746    * revert patch the client
8747    */
8748   private void patchRevertUi() {
8749     this.revertPatches(AppToUpgrade.UI, AppToUpgrade.UI);
8750     boolean patchesReverted = this.revertPatches(AppToUpgrade.API, AppToUpgrade.UI);
8751     if (patchesReverted && (this.grouperInstallerMainFunction == GrouperInstallerMainFunction.patch 
8752         || this.grouperInstallerMainFunction == GrouperInstallerMainFunction.upgrade)) {
8753       System.out.print("Since patches were reverted, you should delete files in your app server work directory,"
8754           + "\n  in tomcat it is named 'work'.  Hit <enter> to continue: ");
8755       readFromStdIn("grouperInstaller.autorun.continueAfterDeleteUiWorkDirectory");
8756     }
8757   }
8758   
8759   /**
8760    * revert patch the client
8761    */
8762   private void patchRevertWs() {
8763     this.revertPatches(AppToUpgrade.WS,AppToUpgrade.WS);
8764     boolean patchesReverted = this.revertPatches(AppToUpgrade.API, AppToUpgrade.WS);
8765     if (patchesReverted && (this.grouperInstallerMainFunction == GrouperInstallerMainFunction.patch 
8766         || this.grouperInstallerMainFunction == GrouperInstallerMainFunction.upgrade)) {
8767       System.out.print("Since patches were reverted, you should delete files in your app server work directory,"
8768           + "\n  in tomcat it is named 'work'.  Hit <enter> to continue: ");
8769       readFromStdIn("grouperInstaller.autorun.continueAfterDeleteWsWorkDirectory");
8770     }
8771   }
8772   
8773   /**
8774    * revert patch the psp
8775    */
8776   private void patchRevertPsp() {
8777     this.revertPatches(AppToUpgrade.PSP, AppToUpgrade.PSP);
8778     this.revertPatches(AppToUpgrade.API, AppToUpgrade.PSP);
8779   }
8780 
8781   /**
8782    * revert patch the pspng
8783    */
8784   private void patchRevertPspng() {
8785     this.revertPatches(AppToUpgrade.PSPNG, AppToUpgrade.PSPNG);
8786     this.revertPatches(AppToUpgrade.API, AppToUpgrade.PSPNG);
8787   }
8788 
8789   /**
8790    * owasp csrf guard file
8791    */
8792   private File owaspCsrfGuardFile;
8793   
8794   /**
8795    * owasp csrf guard base file
8796    */
8797   private File owaspCsrfGuardBaseFile;
8798   
8799   /**
8800    * on an upgrade, compare a new jar and an existing jar and see if needs to be updated, and if so, update it
8801    * @param existingJarFile
8802    * @param newJarFile
8803    * @param printResultIfNotUpgrade
8804    * @param toDir if file not there, copy here.  If null, then go to upgrade existing lib directory string
8805    * @return true if upgraded, false if not
8806    */
8807   private boolean compareAndReplaceJar(File existingJarFile, File newJarFile, boolean printResultIfNotUpgrade, File toDir) {
8808     
8809     if (toDir == null) {
8810       toDir = new File(this.upgradeExistingLibDirectoryString);
8811     }
8812     
8813     if (existingJarFile == null || !existingJarFile.exists()) {
8814       System.out.println(newJarFile.getName() + " is a new file and is being copied to the application lib dir");
8815       existingJarFile = new File(toDir.getAbsoluteFile() + File.separator + newJarFile.getName());
8816       GrouperInstallerUtils.copyFile(newJarFile, existingJarFile, true);
8817       return true;
8818     }
8819 
8820     String existingJarFilePath = existingJarFile.getAbsolutePath();
8821     if (!GrouperInstallerUtils.filePathStartsWith(existingJarFilePath,this.upgradeExistingApplicationDirectoryString)) {
8822       throw new RuntimeException("Why does existing path not start with upgrade path??? " + existingJarFilePath + ", " + this.upgradeExistingApplicationDirectoryString);
8823     }
8824     
8825     String bakJarFileString = this.grouperBaseBakDir + existingJarFilePath.substring(this.upgradeExistingApplicationDirectoryString.length());
8826     File bakJarFile = new File(bakJarFileString);
8827     
8828     String existingVersion = GrouperInstallerUtils.jarVersion(existingJarFile);
8829     String newVersion = GrouperInstallerUtils.jarVersion(newJarFile);
8830     
8831     long existingSize = existingJarFile.length();
8832     long newSize = newJarFile.length();
8833     
8834     if (!GrouperInstallerUtils.equals(existingVersion, newVersion) || existingSize != newSize) {
8835 
8836       //make sure parents exist
8837       GrouperInstallerUtils.createParentDirectories(bakJarFile);
8838       
8839       System.out.println(existingJarFile.getName() + " had version " + existingVersion + " and size " + existingSize + " bytes and is being upgraded to version "
8840           + newVersion + " and size " + newSize + " bytes.\n  It is backed up to " + bakJarFile);
8841 
8842       GrouperInstallerUtils.fileMove(existingJarFile, bakJarFile);
8843       
8844       GrouperInstallerUtils.copyFile(newJarFile, existingJarFile, true);
8845       
8846       return true;
8847     }
8848     
8849     if (printResultIfNotUpgrade) {
8850       System.out.println(existingJarFile.getName() + " is up to date");
8851     }
8852     return false;
8853   }
8854 
8855   /**
8856    * on an upgrade, compare a new file and an existing file and see if needs to be updated, and if so, update it
8857    * @param existingFile
8858    * @param newFile
8859    * @param printResultIfNotUpgrade
8860    * @param toDir if file not there, copy here.  If null, then go to upgrade existing lib directory string
8861    * @return true if upgraded, false if not
8862    */
8863   private boolean compareAndCopyFile(File existingFile, File newFile, boolean printResultIfNotUpgrade, File toDir) {
8864     
8865     if (toDir == null) {
8866       throw new RuntimeException("Which dir to copy to??? " + newFile + ", " + existingFile);
8867     }
8868     
8869     if (existingFile == null || !existingFile.exists()) {
8870       System.out.println(newFile.getName() + " is a new file and is being copied to the application dir: " + toDir.getAbsolutePath());
8871       existingFile = new File(toDir.getAbsoluteFile() + File.separator + newFile.getName());
8872       GrouperInstallerUtils.copyFile(newFile, existingFile, true);
8873       return true;
8874     }
8875 
8876     String existingFilePath = existingFile.getAbsolutePath();
8877     if (!GrouperInstallerUtils.filePathStartsWith(existingFilePath,this.upgradeExistingApplicationDirectoryString)) {
8878       throw new RuntimeException("Why does existing path not start with upgrade path??? " + existingFilePath + ", " + this.upgradeExistingApplicationDirectoryString);
8879     }
8880     
8881     String bakFileString = this.grouperBaseBakDir + existingFilePath.substring(this.upgradeExistingApplicationDirectoryString.length());
8882     File bakFile = new File(bakFileString);
8883     
8884     String existingChecksum = GrouperInstallerUtils.fileSha1(existingFile);
8885     String newChecksum = GrouperInstallerUtils.fileSha1(newFile);
8886     
8887     long existingSize = existingFile.length();
8888     long newSize = newFile.length();
8889     
8890     if (!GrouperInstallerUtils.equals(existingChecksum, newChecksum) || existingSize != newSize) {
8891 
8892       //make sure parents exist
8893       GrouperInstallerUtils.createParentDirectories(bakFile);
8894       
8895       System.out.println(existingFile.getName() + " had checksum " + existingChecksum + " and size " + existingSize + " bytes and is being upgraded to checksum "
8896           + newChecksum + " and size " + newSize + " bytes.\n  It is backed up to " + bakFile);
8897 
8898       GrouperInstallerUtils.fileMove(existingFile, bakFile);
8899       
8900       GrouperInstallerUtils.copyFile(newFile, existingFile, true);
8901       
8902       return true;
8903     }
8904     
8905     if (printResultIfNotUpgrade) {
8906       System.out.println(existingFile.getName() + " is up to date");
8907     }
8908     return false;
8909   }
8910 
8911   /**
8912    * grouper.client.properties
8913    */
8914   private File grouperClientPropertiesFile;
8915   
8916   /**
8917    * grouper.client.base.properties
8918    */
8919   private File grouperClientBasePropertiesFile;
8920   
8921   /**
8922    * grouper.client.example.properties
8923    */
8924   private File grouperClientExamplePropertiesFile;
8925 
8926   /**
8927    * grouperClient.jar
8928    */
8929   private File grouperClientJar;
8930 
8931   /**
8932    * grouper.properties
8933    */
8934   private File grouperPropertiesFile;
8935   
8936   /**
8937    * grouper.base.properties
8938    */
8939   private File grouperBasePropertiesFile;
8940   
8941   /**
8942    * subject.properties
8943    */
8944   private File subjectPropertiesFile;
8945   
8946   /**
8947    * subject.base.properties
8948    */
8949   private File subjectBasePropertiesFile;
8950   
8951   /**
8952    * grouperUtf8.txt
8953    */
8954   private File grouperUtf8File;
8955   
8956   /**
8957    * GSHFileLoad.properties
8958    */
8959   private File gshFileLoadPropertiesFile;
8960   
8961   /**
8962    * groovysh.profile
8963    */
8964   private File groovyshProfileFile;
8965   
8966   /**
8967    * grouper.client.usage.example.txt
8968    */
8969   private File grouperClientUsageExampleFile;
8970   
8971   /**
8972    * grouper.example.properties
8973    */
8974   private File grouperExamplePropertiesFile;
8975 
8976   /**
8977    * grouper.hibernate.properties
8978    */
8979   private File grouperHibernatePropertiesFile;
8980   
8981   /**
8982    * grouper.hibernate.base.properties
8983    */
8984   private File grouperHibernateBasePropertiesFile;
8985   
8986   /**
8987    * grouper.hibernate.example.properties
8988    */
8989   private File grouperHibernateExamplePropertiesFile;
8990 
8991   /**
8992    * grouper-ws.properties
8993    */
8994   private File grouperWsPropertiesFile;
8995   
8996   /**
8997    * grouper-ws.example.properties
8998    */
8999   private File grouperWsBasePropertiesFile;
9000 
9001   /**
9002    * grouper-ws.base.properties
9003    */
9004   private File grouperWsExamplePropertiesFile;
9005   
9006   /**
9007    * ehcache.xml
9008    */
9009   private File ehcacheFile;
9010 
9011   /**
9012    * ehcache.example.xml
9013    */
9014   private File ehcacheExampleFile;
9015   
9016   /**
9017    * grouper-loader.properties
9018    */
9019   private File grouperLoaderPropertiesFile;
9020   
9021   /**
9022    * grouper-loader.base.properties
9023    */
9024   private File grouperLoaderBasePropertiesFile;
9025   
9026   /**
9027    * grouper.cache.properties
9028    */
9029   private File grouperCachePropertiesFile;
9030   
9031   /**
9032    * grouper.cache.base.properties
9033    */
9034   private File grouperCacheBasePropertiesFile;
9035   
9036   /**
9037    * grouper-loader.example.properties
9038    */
9039   private File grouperLoaderExamplePropertiesFile;
9040 
9041   /**
9042    * grouper.jar
9043    */
9044   private File grouperJar;
9045 
9046   
9047   /**
9048    * find a classpath file on classpath by resourceName
9049    * @param resourceName resource name of file
9050    * @param exceptionIfNotFound 
9051    * @return the file or null if not exception if not found
9052    */
9053   private File findClasspathFile(String resourceName, boolean exceptionIfNotFound) {
9054     
9055     Set<String> fileNamesTried = new LinkedHashSet<String>();
9056     
9057     File file = new File(this.upgradeExistingApplicationDirectoryString + "classes" + File.separator + resourceName);
9058     if (file.exists()) {
9059       return file;
9060     }
9061     
9062     fileNamesTried.add(file.getAbsolutePath());
9063     
9064     file = new File(this.upgradeExistingApplicationDirectoryString + "conf" + File.separator + resourceName);
9065     if (file.exists()) {
9066       return file;
9067     }
9068 
9069     fileNamesTried.add(file.getAbsolutePath());
9070 
9071     //these could be in this location
9072     if (GrouperInstallerUtils.equals("nav.properties", resourceName) 
9073         || GrouperInstallerUtils.equals("media.properties", resourceName)) {
9074       file = new File(this.upgradeExistingApplicationDirectoryString + "WEB-INF" + File.separator 
9075           + "classes" + File.separator + "resources" + File.separator + "grouper" + File.separator + resourceName);
9076       if (file.exists()) {
9077         return file;
9078       }
9079       
9080       fileNamesTried.add(file.getAbsolutePath());
9081     }
9082     
9083     file = new File(this.upgradeExistingApplicationDirectoryString + "WEB-INF" + File.separator + "classes" 
9084         + File.separator + resourceName);
9085     if (file.exists()) {
9086       return file;
9087     }
9088 
9089     fileNamesTried.add(file.getAbsolutePath());
9090     
9091     file = new File(this.upgradeExistingApplicationDirectoryString + resourceName);
9092     if (file.exists()) {
9093       return file;
9094     }
9095 
9096     fileNamesTried.add(file.getAbsolutePath());
9097     
9098     if (exceptionIfNotFound) {
9099       throw new RuntimeException("Cant find file, looked in: " + GrouperInstallerUtils.join(fileNamesTried.iterator(), ", "));
9100     }
9101     
9102     return null;
9103   }
9104 
9105   /**
9106    * lib dirs where libs might be in this.upgradeExistingApplicationDirectoryString
9107    */
9108   private static List<String> libDirs = GrouperInstallerUtils.toList(
9109       "lib" + File.separator, 
9110       "WEB-INF" + File.separator + "lib" + File.separator,
9111       "lib" + File.separator + "grouper" + File.separator,
9112       "lib" + File.separator + "custom" + File.separator,
9113       "lib" + File.separator + "jdbcSamples" + File.separator,
9114       "dist" + File.separator + "lib" + File.separator,
9115       "");
9116 
9117   /**
9118    * get all library files
9119    * @param appDir 
9120    * @return the list of files
9121    */
9122   private List<File> findAllLibraryFiles(String appDir) {
9123     
9124     if (!appDir.endsWith("/") && !appDir.endsWith("\\")) {
9125       appDir = appDir + File.separator;
9126     }
9127     
9128     List<File> result = new ArrayList<File>();
9129     for (String libDir : libDirs) {
9130 
9131       File dir = new File(appDir + libDir);
9132       if (dir.exists() && dir.isDirectory()) {
9133         for (File file : dir.listFiles()) {
9134           if (file.getName().endsWith(".jar")) {
9135             result.add(file);
9136           }
9137         }
9138       }
9139       
9140     }
9141     return result;
9142   }
9143 
9144   /**
9145    * if file is there, return it.  if not a jar, return original
9146    * find a library file on lib dir by lib name, if there return it
9147    * otherwise just return the original file name.  fix the name if not correct
9148    * @param originalThoughtLocation 
9149    * @param originalAppToUpgrade
9150    * @return the file or null if not exception if not found
9151    */
9152   private File fixLibraryFileIfFoundAndDifferent(File originalThoughtLocation, AppToUpgrade originalAppToUpgrade) {
9153     
9154     if (originalThoughtLocation == null || (originalThoughtLocation.exists() && originalThoughtLocation.isFile())) {
9155       return originalThoughtLocation;
9156     }
9157     
9158     if (!originalThoughtLocation.getAbsolutePath().endsWith(".jar")) {
9159       return originalThoughtLocation;
9160     }
9161     
9162     File foundLibraryFile = findLibraryFile(originalThoughtLocation.getName(), false);
9163     if (foundLibraryFile != null && foundLibraryFile.exists() && foundLibraryFile.isFile()) {
9164       return foundLibraryFile;
9165     }
9166     
9167     if (!originalAppToUpgrade.isApiOrganized()) {
9168       
9169       // is in WEB-INF/lib/something.jar
9170       if (GrouperInstallerUtils.equals("lib", originalThoughtLocation.getParentFile().getName())) {
9171         return originalThoughtLocation;
9172       }
9173       
9174       if (GrouperInstallerUtils.equals("lib", originalThoughtLocation.getParentFile().getParentFile().getName())) {
9175         return new File(originalThoughtLocation.getParentFile().getParentFile().getAbsoluteFile() + File.separator + originalThoughtLocation.getName());
9176       }
9177       
9178       return originalThoughtLocation;
9179     }
9180     
9181     //is api
9182     if (!GrouperInstallerUtils.equals("lib", originalThoughtLocation.getParentFile().getName())
9183         && GrouperInstallerUtils.equals("lib", originalThoughtLocation.getParentFile().getParentFile().getName())) {
9184       return originalThoughtLocation;
9185     }
9186       
9187     if (GrouperInstallerUtils.equals("lib", originalThoughtLocation.getParentFile().getName())) {
9188       return new File(originalThoughtLocation.getParentFile().getAbsoluteFile() + File.separator + "grouper" + File.separator + originalThoughtLocation.getName());
9189     }
9190     
9191     // not sure what to do here
9192     return originalThoughtLocation;
9193   }
9194 
9195   /**
9196    * find a library file on lib dir by libName
9197    * @param libName lib name of file
9198    * @param exceptionIfNotFound 
9199    * @return the file or null if not exception if not found
9200    */
9201   private File findLibraryFile(String libName, boolean exceptionIfNotFound) {
9202     
9203     Set<String> fileNamesTried = new LinkedHashSet<String>();
9204 
9205     for (String libDir : libDirs) {
9206 
9207       File file = new File(this.upgradeExistingApplicationDirectoryString + libDir + libName);
9208       if (file.exists()) {
9209         return file;
9210       }
9211       
9212       fileNamesTried.add(file.getAbsolutePath());
9213       
9214     }
9215     
9216     if (exceptionIfNotFound) {
9217       throw new RuntimeException("Cant find file, looked in: " + GrouperInstallerUtils.join(fileNamesTried.iterator(), ", "));
9218     }
9219     
9220     return null;
9221   }
9222   
9223   /**
9224    * 
9225    */
9226   private void mainInstallContainerLogic() {
9227     
9228     System.out.print("The Grouper Installer will install a \"maturity level 0\" Grouper environment using Docker. You do not have to use Docker to run Grouper, but you need Docker for this Grouper Installer. Do you have Docker installed and running? (t|f) [t]: ");
9229     
9230     boolean dockerInstalledAndRunning = readFromStdInBoolean(true, "");
9231     
9232     if (!dockerInstalledAndRunning) {
9233       System.out.println("Please install and run docker before proceeding. Thanks! ");
9234       return;
9235     }
9236     
9237     boolean validBaseDirectoryFound = false;
9238     String path = null;
9239     do {
9240       File grouperContainerBaseDirectory = new File(new File("").getAbsolutePath());
9241       
9242       System.out.print("Where do you want your host grouper container base directory (e.g. /opt/grouperContainer)? ["+grouperContainerBaseDirectory.getAbsolutePath()+"]: ");
9243       String localGrouperContainerBaseDirectoryString = readFromStdIn("Placeholder");
9244       if (!GrouperInstallerUtils.isBlank(localGrouperContainerBaseDirectoryString)) {
9245         File grouperContainerBaseDirectoryFile = new File(localGrouperContainerBaseDirectoryString);
9246         if (!grouperContainerBaseDirectoryFile.exists() || !grouperContainerBaseDirectoryFile.isDirectory()) { 
9247           System.out.println("Error: cant find directory: '" + grouperContainerBaseDirectoryFile.getAbsolutePath() + "'");
9248         } else {
9249           path = grouperContainerBaseDirectoryFile.getAbsolutePath();
9250           validBaseDirectoryFound = true;
9251         }
9252       } else {
9253         path = grouperContainerBaseDirectory.getAbsolutePath();
9254         validBaseDirectoryFound = true;
9255       }
9256       
9257     } while (validBaseDirectoryFound == false);
9258     
9259     
9260     // create README.txt file at the path
9261     File readmeFile = new File(path + File.separator + "README.txt");
9262     if (readmeFile.exists()) {
9263       String newFileName = "README_" + new Date().toString().replace(" ", "_") + ".txt";
9264       System.out.println("README.txt already exists. Going to rename to "+newFileName);
9265       readmeFile.renameTo(new File(path + File.separator + newFileName));
9266       readmeFile = new File(path + File.separator + "README.txt");
9267     }
9268         
9269     GrouperInstallerUtils.fileCreate(readmeFile);
9270     
9271     // create logs directory
9272     StringBuilder contentToWrite = new StringBuilder();
9273     contentToWrite.append("Create logs directory in "+path);
9274     contentToWrite.append("\n\n");
9275     contentToWrite.append("\n\n");
9276     try {      
9277       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
9278     } catch (Exception e) {
9279       System.out.println("Could not write to README.txt file.");
9280     }
9281     
9282     File logsDirectory = new File(path+File.separator+"logs"+File.separator+"nothing");
9283     GrouperInstallerUtils.createParentDirectories(logsDirectory);
9284     
9285     File logsDirectoryOnly = new File(path+File.separator+"logs");
9286     
9287     // run chmod o+w for logs directory
9288     contentToWrite = new StringBuilder();
9289     contentToWrite.append("Run chmod o+w for "+logsDirectoryOnly.getAbsolutePath());
9290     contentToWrite.append("\n\n");
9291     contentToWrite.append("\n\n");
9292     try {      
9293       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
9294     } catch (Exception e) {
9295       System.out.println("Could not write to README.txt file.");
9296     }
9297     
9298     List<String> openWriteCommands = GrouperInstallerUtils.toList("chmod", "o+w", 
9299         logsDirectoryOnly.getAbsolutePath() + File.separator);
9300     
9301     System.out.println("Making logs directory o+w so that logs can be written from inside the container: " + convertCommandsIntoCommand(openWriteCommands) + "\n");
9302 
9303     String errorMessageOnChangingLogsDirectoryPermissions = "";
9304     boolean errorOnChangingLogsDirectoryPermissions = false;
9305     try {
9306       CommandResult openWriteCommandResult = GrouperInstallerUtils.execCommand(
9307           GrouperInstallerUtils.toArray(openWriteCommands, String.class), true, true, null, 
9308           new File("."), null, true);
9309       
9310       if (openWriteCommandResult.getExitCode() != 0) {
9311         errorMessageOnChangingLogsDirectoryPermissions = openWriteCommandResult.getErrorText();
9312         errorOnChangingLogsDirectoryPermissions = true;
9313       } 
9314       
9315     } catch (Throwable e) {
9316       errorOnChangingLogsDirectoryPermissions = true;
9317     }
9318     
9319     if (errorOnChangingLogsDirectoryPermissions) {
9320       System.out.println("Could not change permissions on logs directory at "+logsDirectoryOnly.getAbsolutePath());
9321       if (GrouperInstallerUtils.isNotBlank(errorMessageOnChangingLogsDirectoryPermissions)) {
9322         System.out.println("Received error message: "+errorMessageOnChangingLogsDirectoryPermissions+ " ");
9323       }
9324       return;
9325     }
9326     
9327     // create log4j.properties file in <path>/slashRoot/opt/grouper/grouperWebapp/WEB-INF/classes
9328     File classesDir = new File(path+File.separator+"slashRoot"+File.separator+"opt"+File.separator+"grouper"
9329         +File.separator+"grouperWebapp"+File.separator+"WEB-INF"+File.separator+"classes");
9330     
9331     File log4jPropertiesFile = new File(classesDir.getAbsolutePath()+File.separator+"log4j.properties");
9332     
9333     contentToWrite = new StringBuilder();
9334     contentToWrite.append("Create log4j.properties file in "+log4jPropertiesFile.getAbsolutePath());
9335     contentToWrite.append("\n\n");
9336     contentToWrite.append("Copy the content from https://spaces.at.internet2.edu/display/Grouper/Install+the+Grouper+v2.5+container+with+maturity+level+0+using+installer");
9337     contentToWrite.append("\n\n");
9338     contentToWrite.append("\n\n");
9339     try {      
9340       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
9341     } catch (Exception e) {
9342       System.out.println("Could not write to README.txt file.");
9343     }
9344     
9345     GrouperInstallerUtils.createParentDirectories(log4jPropertiesFile);
9346     boolean reuseLog4jPropertiesFile = false;
9347     while(true) {
9348       if (log4jPropertiesFile.exists()) {
9349         System.out.print("log4j.properties already exists at '"+log4jPropertiesFile.getParent()+"' ");
9350         System.out.print("Do you want to reuse it (t|f) [t]: ");
9351         reuseLog4jPropertiesFile = readFromStdInBoolean(true, "Placeholder");
9352         if (reuseLog4jPropertiesFile) {
9353           System.out.println("Going to reuse existing log4j.properties file. ");
9354           break;
9355         } else {
9356           System.out.print("Delete log4j.properties and press <return> to continue ");
9357           readFromStdIn("nothing");
9358           log4jPropertiesFile = new File(path+File.separator+"conf"+File.separator+"log4j.properties");
9359           continue;
9360         }
9361       } else {
9362         GrouperInstallerUtils.fileCreate(log4jPropertiesFile);
9363         break;
9364       }
9365     }
9366     
9367     try {
9368       InputStream in = getClass().getResourceAsStream("/log4j.sample.properties"); 
9369       BufferedReader reader = new BufferedReader(new InputStreamReader(in));
9370       StringBuilder log4jContent = new StringBuilder();
9371       String line;
9372       while( (line = reader.readLine()) != null) {
9373         log4jContent.append(line);
9374         log4jContent.append("\n");
9375       }
9376       Files.write(Paths.get(log4jPropertiesFile.getAbsolutePath()), log4jContent.toString().getBytes(), StandardOpenOption.APPEND);
9377     } catch (Exception e) {
9378       System.out.println("Could not write content to "+log4jPropertiesFile.getAbsolutePath());
9379       System.out.println("Go to https://spaces.at.internet2.edu/display/Grouper/Install+the+Grouper+v2.5+container+with+maturity+level+0+using+installer to copy the log4j.properties section manually. ");
9380       System.out.print("press <return> to continue ");
9381       readFromStdIn("Placeholder");
9382     }
9383     
9384     
9385     String dockerImageVersion = GrouperInstallerUtils.propertiesValue("grouperInstaller.docker.image.version", false);
9386     
9387     if (GrouperInstallerUtils.isBlank(dockerImageVersion)) {
9388       dockerImageVersion = getClass().getPackage().getImplementationVersion();
9389     }
9390     
9391     if (GrouperInstallerUtils.isBlank(dockerImageVersion)) {
9392       dockerImageVersion = GrouperInstallerUtils.propertiesValue("grouper.version", true);
9393     }
9394     
9395     contentToWrite = new StringBuilder("Make sure docker is installed and running. Run the following command to check if docker is installed.");
9396     contentToWrite.append("\n\n");
9397     contentToWrite.append("which docker");
9398     contentToWrite.append("\n\n");
9399     contentToWrite.append("If docker is not installed, go to: https://docs.docker.com/install/ and select the correct platform and follow the instructions. ");
9400     contentToWrite.append("\n\n");
9401     contentToWrite.append("\n\n");
9402     
9403     contentToWrite.append("Run the following command to check if docker is running");
9404     contentToWrite.append("\n\n");
9405     contentToWrite.append("docker info");
9406     
9407     contentToWrite.append("\n\n");
9408     contentToWrite.append("\n\n");
9409     contentToWrite.append("Run the following command to start docker if it's not running already. Command might vary based on the platform.");
9410     contentToWrite.append("\n\n");
9411     contentToWrite.append("sudo service docker start");
9412     
9413     contentToWrite.append("\n\n");
9414     contentToWrite.append("\n\n");
9415     
9416     try {      
9417       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
9418     } catch (Exception e) {
9419       System.out.println("Could not write to README.txt file.");
9420     }
9421     
9422     List<String> commands = new ArrayList<String>();
9423     commands.add(shCommand());
9424     commands.add("-c");
9425     commands.add("which docker");
9426     
9427     String dockerLocation = null;
9428     boolean errorDetectingDocker = false;
9429     String errorMessageDetectingDocker = null;
9430     
9431     try {
9432       CommandResult commandResult = GrouperInstallerUtils.execCommand(
9433           GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
9434           new File("."), null, false, false, true);
9435       
9436       if (commandResult.getExitCode() != 0) {
9437         errorMessageDetectingDocker = commandResult.getErrorText();
9438         errorDetectingDocker = true;
9439       } else {
9440         dockerLocation = commandResult.getOutputText();
9441       }
9442       
9443     } catch (Throwable e) {
9444       errorDetectingDocker = true;
9445     }
9446     
9447     if (errorDetectingDocker) {
9448       System.out.println("Could not detect if docker is installed with command 'which docker' ");
9449       if (GrouperInstallerUtils.isNotBlank(errorMessageDetectingDocker)) {
9450         System.out.println("Received error message: "+errorMessageDetectingDocker+ " ");
9451       }
9452       
9453       System.out.println("Make sure you are running the installer as the user that runs docker ");
9454       
9455       System.out.print("If you have docker installed, enter 't' otherwise enter 'f': ");
9456       boolean isDockerInstalled = readFromStdInBoolean(null, "Placeholder");
9457       
9458       if (isDockerInstalled) {
9459         System.out.print("Please enter the full absolute path to docker. eg: /usr/local/bin/docker ");
9460         dockerLocation = readFromStdIn("Placeholder");
9461         
9462         while (true) {
9463           if (GrouperInstallerUtils.isBlank(dockerLocation) || !new File(dockerLocation).exists()) {
9464             System.out.print("Path is invalid. Please try again. ");
9465             dockerLocation = readFromStdIn("Placeholder");
9466           } else {
9467             break;
9468           }
9469         }
9470       } else {
9471         System.out.print("Please install docker first and try again. ");
9472         return;
9473       }
9474     } else {
9475       // docker is installed
9476       System.out.println("We detected docker is installed at: "+dockerLocation);
9477       System.out.print("Is the path above correct? (t|f) [t]: ");
9478       boolean correctDockerLocationDetected = readFromStdInBoolean(true, "Placeholder");
9479       
9480       if (correctDockerLocationDetected == false) {
9481         System.out.print("Please enter the full absolute path to docker. eg: /usr/local/bin/docker : ");
9482         dockerLocation = readFromStdIn("Placeholder");
9483         
9484         while (true) {
9485           if (GrouperInstallerUtils.isBlank(dockerLocation) || !new File(dockerLocation).exists()) {
9486             System.out.print("Path is invalid. Please try again. ");
9487             dockerLocation = readFromStdIn("Placeholder");
9488           } else {
9489             break;
9490           }
9491         }
9492       }
9493     }
9494     
9495     
9496     dockerLocation = dockerLocation.trim();
9497     
9498     // check if docker is running or not. 
9499     
9500     System.out.println("Going to check if docker is running. ");
9501     boolean dockerIsRunning = false;
9502     CommandResult commandResult = null;
9503     while (true) {
9504       try {
9505         commandResult = GrouperInstallerUtils.execCommand(
9506             new String[] {shCommand(), "-c", dockerLocation + " info"}, true, true, null, 
9507             new File("."), null, false, false, true);
9508         if (commandResult.getExitCode() == 0) {
9509           dockerIsRunning = true;
9510         }
9511       } catch (Exception e) {}
9512       
9513       if (dockerIsRunning == false) {
9514         
9515         System.out.print("Could not determine if docker is running. You can run 'docker info' to check if docker is running. Do you have docker running? (t|f): ");
9516         boolean isDockerRunning = readFromStdInBoolean(null, "Placeholder");
9517         
9518         if (isDockerRunning) {
9519           break;
9520         } else {
9521           System.out.print("Start docker and press <return> to continue ");
9522           readFromStdIn("Placeholder");
9523           continue;
9524         }
9525         
9526       } else {
9527         System.out.println("docker is running. ");
9528         System.out.println(commandResult.getOutputText());
9529         break;
9530       }
9531     }
9532     
9533     
9534     // if there's a container with name gsh, ws, ui, etc is already running. ask user to stop them first
9535     
9536     contentToWrite = new StringBuilder();
9537     contentToWrite.append("Run the following command to view the containers names");
9538     contentToWrite.append("\n\n");
9539     contentToWrite.append("docker ps --all --format \"{{.Names}}\" ");
9540     contentToWrite.append("\n");
9541     contentToWrite.append("If you have gsh, ws, grouper or ui containers already there. Please stop them, remove them and then continue.");
9542     contentToWrite.append("\n");
9543     
9544     contentToWrite.append("To stop a running container, run the following command. ");
9545     contentToWrite.append("\n");
9546     contentToWrite.append("docker kill <container name>");
9547     contentToWrite.append("\n");
9548     contentToWrite.append("You might want to add -f flag to docker kill command if unable to stop.");
9549     contentToWrite.append("\n");
9550     contentToWrite.append("To remove the container, run the following command.");
9551     contentToWrite.append("\n");
9552     contentToWrite.append("docker rm <container name>");
9553     contentToWrite.append("\n");
9554     contentToWrite.append("You might want to add -f flag to docker rm command if unable to remove.");
9555     contentToWrite.append("\n\n");
9556     contentToWrite.append("\n\n");
9557     
9558     try {      
9559       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
9560     } catch (Exception e) {
9561       System.out.println("Could not write to README.txt file.");
9562     }
9563     
9564     System.out.println("Going to check if gsh, ws, grouper-ui, or ui containers already exist.");
9565     List<String> conflictingNames = new ArrayList<String>();
9566     boolean conflictingNamesRanSuccessfully = false;
9567     try {
9568       commandResult = GrouperInstallerUtils.execCommand(
9569           new String[] {shCommand(), "-c", dockerLocation + " ps --all --format \"{{.Names}}\""}, true, true, null, 
9570           new File("."), null, false, false, true);
9571       if (commandResult.getExitCode() == 0) {
9572         conflictingNamesRanSuccessfully = true;
9573         String containerNamesString = commandResult.getOutputText();
9574         if (GrouperInstallerUtils.isNotBlank(containerNamesString)) {
9575           String[] containerNames = containerNamesString.split("\n");
9576           
9577           List<String> containersThatCanCauseConflict = Arrays.asList("gsh", "ui", "ws", "grouper", "grouper-ui");
9578           
9579           for (String containerName: containerNames) {
9580             if (containersThatCanCauseConflict.contains(containerName)) {
9581               conflictingNames.add(containerName);
9582             }
9583           }
9584         }
9585       }
9586     } catch (Exception e) {}
9587     
9588     if (conflictingNamesRanSuccessfully == false) {
9589       System.out.println("There was an error trying to figure out if gsh, ui, or ws containers already exist.");
9590       System.out.println("Run 'docker ps --all --format \"{{.Names}}\"' to see if gsh, ws, grouper, grouper-ui, or ui already exist. Please delete them before proceeding. ");
9591       System.out.println("Run docker rm -f <container name> to force remove a docker container");
9592     }
9593     
9594     if (conflictingNames.size() > 0) {
9595       System.out.println("We found that containers with names "+String.join(", ", conflictingNames) + " already exist. Please delete them before proceeding. ");
9596       System.out.println("Command to delete a docker container is 'docker rm <container name>'. Use -f flag to force remove. ");
9597       System.out.print("press <return> once you have deleted the conflicting containers. ");
9598       readFromStdIn("Placeholder");
9599     }
9600     if (conflictingNamesRanSuccessfully && conflictingNames.size() == 0) {
9601       System.out.println("No conflicting containers found. ");
9602     }
9603     
9604     // pull grouper docker image
9605     contentToWrite = new StringBuilder();
9606     contentToWrite.append("Pull grouper docker image by running the following command. ");
9607     contentToWrite.append("\n\n");
9608     contentToWrite.append("docker pull i2incommon/grouper:"+dockerImageVersion);
9609     contentToWrite.append("\n\n");
9610     contentToWrite.append("\n\n");
9611     
9612     try {      
9613       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
9614     } catch (Exception e) {
9615       System.out.println("Could not write to README.txt file.");
9616     }
9617     
9618     System.out.println("Going to pull grouper docker image: i2incommon/grouper:"+dockerImageVersion);
9619     boolean pulledDockerImage = false;
9620     try {
9621       
9622       String dockerPullCommand = dockerLocation + " pull i2incommon/grouper:"+dockerImageVersion;
9623       
9624       commandResult = GrouperInstallerUtils.execCommand(
9625           new String[] {shCommand(), "-c", dockerPullCommand}, true, true, null, 
9626           new File("."), null, false, true, true);
9627       
9628       if (commandResult.getExitCode() == 0) {
9629         pulledDockerImage = true;
9630       }
9631       
9632       if (commandResult.getOutputText() != null) {
9633         System.out.println(commandResult.getOutputText());
9634       }
9635       
9636     } catch (Exception e) {
9637       // TODO: handle exception
9638     }
9639     
9640     if(pulledDockerImage == false) {
9641       System.out.println("Could not pull grouper docker image. Pull it manually by running: docker pull i2incommon/grouper:"+dockerImageVersion);
9642       System.out.print("press <return> when done ");
9643       readFromStdIn("Placeholder");
9644     }
9645     
9646     // create slashRoot directory
9647     contentToWrite = new StringBuilder();
9648     contentToWrite.append("Create slashRoot directory in "+path);
9649     contentToWrite.append("\n\n");
9650     try {      
9651       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
9652     } catch (Exception e) {
9653       System.out.println("Could not write to README.txt file.");
9654     }
9655     
9656     // create morphString.properties file in slashRoot/opt/grouper/grouperWebapp/WEB-INF/classes/
9657     contentToWrite = new StringBuilder();
9658     contentToWrite.append("Create morphString.properties file in "+path+"/slashRoot/opt/grouper/grouperWebapp/WEB-INF/classes/");
9659     contentToWrite.append("\n");
9660     contentToWrite.append("Add the following lines to morphString.properties file. Replace the placeholders below with actual values");
9661     contentToWrite.append("\n");
9662     contentToWrite.append("encrypt.key = <random alphanumeric key with minimum 8 characters>");
9663     contentToWrite.append("\n\n");
9664     contentToWrite.append("\n\n");
9665     
9666     try {      
9667       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
9668     } catch (Exception e) {
9669       System.out.println("Could not write to README.txt file.");
9670     }
9671     
9672     File morphStringPropertiesFile = new File(classesDir+File.separator+"morphString.properties");
9673     
9674     GrouperInstallerUtils.createParentDirectories(morphStringPropertiesFile);
9675     boolean reuseMorphStringPropertiesFile = false;
9676     while(true) {
9677       if (morphStringPropertiesFile.exists()) {
9678         System.out.println("morphString.properties already exists at "+morphStringPropertiesFile.getParent()+" ");
9679         System.out.print("Do you want to reuse it (t|f) [t]: ");
9680         reuseMorphStringPropertiesFile = readFromStdInBoolean(true, "Placeholder");
9681         if (reuseMorphStringPropertiesFile) {
9682           System.out.println("Going to reuse existing morphString.properties file. ");
9683           break;
9684         } else {
9685           System.out.print("Delete morphString.properties and press <return> to continue ");
9686           readFromStdIn("nothing");
9687           morphStringPropertiesFile = new File(path+File.separator+"conf"+File.separator+"morphString.properties");
9688           continue;
9689         }
9690       } else {
9691         GrouperInstallerUtils.fileCreate(morphStringPropertiesFile);
9692         break;
9693       }
9694     }
9695     
9696     if (reuseMorphStringPropertiesFile == false) {
9697       
9698       String validCharactersMorphString = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
9699       SecureRandom sr = new SecureRandom();
9700       
9701       StringBuilder morphStringBuilder = new StringBuilder(20);
9702       for( int i = 0; i < 20; i++ ) {
9703         morphStringBuilder.append( validCharactersMorphString.charAt(sr.nextInt(validCharactersMorphString.length())));
9704       }
9705       
9706       System.out.print("Do you want to use the randomly generated morphString key? (" +  morphStringBuilder.toString()+") (t|f) [t]: ");
9707       boolean useAutoGeneratedMorphKey = readFromStdInBoolean(true, "Placeholder");
9708       String morphStringPasswd = null;
9709       if (useAutoGeneratedMorphKey == false) {
9710         System.out.print("Enter morphString key. Minimum 8 characters required: ");
9711         while (true) {          
9712           String manualMorphKey = readFromStdIn("Placeholder");
9713           if (GrouperInstallerUtils.isNotBlank(manualMorphKey) && manualMorphKey.trim().length() >= 8) {
9714             morphStringPasswd = manualMorphKey.trim();
9715             break;
9716           } else {
9717             System.out.print("morphString key is invalid. Minimum 8 characters required. Please try again: ");
9718             continue;
9719           } 
9720         }
9721         
9722       } else {
9723         morphStringPasswd = morphStringBuilder.toString();
9724       }
9725       
9726       editPropertiesFile(morphStringPropertiesFile, "encrypt.key", morphStringPasswd, false);
9727     }
9728     
9729     Properties morphStringProperties = GrouperInstallerUtils.propertiesFromFile(morphStringPropertiesFile);
9730     
9731     // create grouper.hibernate.properties file
9732     contentToWrite = new StringBuilder();
9733     contentToWrite.append("Create grouper.hibernate.properties file in " +path+"/" + classesDir.getAbsolutePath());
9734     contentToWrite.append("\n");
9735     contentToWrite.append("Add the following lines to grouper.hibernate.properties file. Replace the placeholders below with actual values");
9736     contentToWrite.append("\n");
9737     contentToWrite.append("hibernate.connection.url = <db url> eg: jdbc:mysql://localhost:3306/grouper");
9738     contentToWrite.append("\n");
9739     contentToWrite.append("hibernate.connection.username = <user> eg: root");
9740     contentToWrite.append("\n");
9741     contentToWrite.append("hibernate.connection.password = <morph string encrypted password> eg: 86asd9f87a9sdf87a9s78df97");
9742     contentToWrite.append("\n");
9743     
9744     contentToWrite.append("grouper.is.ui.basicAuthn = true");
9745     contentToWrite.append("\n");
9746     
9747     contentToWrite.append("grouper.is.ws.basicAuthn = true");
9748     contentToWrite.append("\n");
9749     contentToWrite.append("\n");
9750     contentToWrite.append("\n");
9751     
9752     try {      
9753       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
9754     } catch (Exception e) {
9755       System.out.println("Could not write to README.txt file.");
9756     }
9757     
9758     System.out.println("Going to create grouper.hibernate.properties file in " + path + File.separator+ classesDir.getAbsolutePath());
9759     
9760     File grouperHibernatePropertiesFile = new File(classesDir+File.separator+"grouper.hibernate.properties");
9761     boolean reuseHibernatePropertiesFile = false;
9762     while(true) {
9763       GrouperInstallerUtils.createParentDirectories(grouperHibernatePropertiesFile);
9764       if (grouperHibernatePropertiesFile.exists()) {
9765         System.out.println("grouper.hibernate.properties already exists at "+grouperHibernatePropertiesFile.getParent()+" ");
9766         System.out.print("Do you want to reuse it (t|f) [t]: ");
9767         reuseHibernatePropertiesFile = readFromStdInBoolean(true, "Placeholder");
9768         if (reuseHibernatePropertiesFile) {
9769           System.out.println("Going to reuse existing grouper.hibernate.properties file. ");
9770           break;
9771         } else {
9772           System.out.print("Delete grouper.hibernate.properties and press <return> to continue ");
9773           readFromStdIn("nothing");
9774           grouperHibernatePropertiesFile = new File(classesDir+File.separator+"grouper.hibernate.properties");
9775           continue;
9776         }
9777       } else {
9778         GrouperInstallerUtils.fileCreate(grouperHibernatePropertiesFile);
9779         break;
9780       }
9781     }
9782     
9783     // create a blank grouper.client.properties
9784     contentToWrite = new StringBuilder();
9785     contentToWrite.append("Create a blank grouper.client.properties file in " +path+File.separator+classesDir.getAbsolutePath());
9786     contentToWrite.append("\n");
9787     contentToWrite.append("\n");
9788     contentToWrite.append("\n");
9789     
9790     try {      
9791       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
9792     } catch (Exception e) {
9793       System.out.println("Could not write to README.txt file.");
9794     }
9795     File grouperClientPropertiesFile = new File(classesDir+File.separator+"grouper.client.properties");
9796     boolean reuseClientPropertiesFile = false;
9797     while(true) {
9798       GrouperInstallerUtils.createParentDirectories(grouperClientPropertiesFile);
9799       if (grouperClientPropertiesFile.exists()) {
9800         System.out.println("grouper.client.properties already exists at "+grouperClientPropertiesFile.getParent()+" ");
9801         System.out.print("Do you want to reuse it (t|f) [t]: ");
9802         reuseClientPropertiesFile = readFromStdInBoolean(true, "Placeholder");
9803         if (reuseClientPropertiesFile) {
9804           System.out.println("Going to reuse existing grouper.client.properties file. ");
9805           break;
9806         } else {
9807           System.out.print("Delete grouper.client.properties and press <return> to continue ");
9808           readFromStdIn("nothing");
9809           grouperClientPropertiesFile = new File(classesDir+File.separator+"grouper.client.properties");
9810           continue;
9811         }
9812       } else {
9813         GrouperInstallerUtils.fileCreate(grouperClientPropertiesFile);
9814         break;
9815       }
9816     }
9817     
9818     // create a blank subject.properties
9819     contentToWrite = new StringBuilder();
9820     contentToWrite.append("Create a blank subject.properties file in " +path+File.separator+classesDir.getAbsolutePath());
9821     contentToWrite.append("\n");
9822     contentToWrite.append("\n");
9823     contentToWrite.append("\n");
9824     
9825     try {      
9826       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
9827     } catch (Exception e) {
9828       System.out.println("Could not write to README.txt file.");
9829     }
9830     File grouperSubjectPropertiesFile = new File(classesDir+File.separator+"subject.properties");
9831     boolean reuseSubjectPropertiesFile = false;
9832     while(true) {
9833       GrouperInstallerUtils.createParentDirectories(grouperSubjectPropertiesFile);
9834       if (grouperSubjectPropertiesFile.exists()) {
9835         System.out.println("subject.properties already exists at "+grouperSubjectPropertiesFile.getParent()+" ");
9836         System.out.print("Do you want to reuse it (t|f) [t]: ");
9837         reuseSubjectPropertiesFile = readFromStdInBoolean(true, "Placeholder");
9838         if (reuseSubjectPropertiesFile) {
9839           System.out.println("Going to reuse existing subject.properties file. ");
9840           break;
9841         } else {
9842           System.out.print("Delete subject.properties and press <return> to continue ");
9843           readFromStdIn("nothing");
9844           grouperSubjectPropertiesFile = new File(classesDir+File.separator+"subject.properties");
9845           continue;
9846         }
9847       } else {
9848         GrouperInstallerUtils.fileCreate(grouperSubjectPropertiesFile);
9849         break;
9850       }
9851     }
9852     
9853     //####################################
9854     //ask about database
9855     if (reuseHibernatePropertiesFile == false) {
9856       System.out.print("Database setup");
9857 
9858       System.out.println("\n##################################\n");
9859       System.out.println("Example mysql URL: jdbc:mysql://1.2.3.4:3306/grouper?useSSL=false");
9860       System.out.println("Example oracle URL: jdbc:oracle:thin:@server.school.edu:1521:sid");
9861       System.out.println("Example postgres URL: jdbc:postgresql://1.2.3.4:5432/database");
9862       System.out.print("\nEnter the database URL: ");
9863       String newDbUrl = readFromStdIn("grouperInstaller.autorun.dbUrl");
9864       if (!GrouperInstallerUtils.isBlank(newDbUrl)) {
9865         this.dbUrl = newDbUrl;
9866         if (newDbUrl.contains("postgresql")) {
9867           System.out.println("Note: you need to change the search sql in the jdbc source in the grouperApi/conf/sources.xml... the change is in the comments in that file");
9868           for (int i=0;i<3;i++) {
9869             System.out.print("Ready to continue? (t|f)? [t] ");
9870             boolean shouldContinue = readFromStdInBoolean(true, "grouperInstaller.autorun.dbContinueAfterChangeSourcesXmlForPostgresSqlServer");
9871             if (shouldContinue) {
9872               break;
9873             }
9874           }
9875         }
9876         
9877         if (newDbUrl.contains("oracle")) {
9878           
9879           System.out.print("Have you reviewed and agreed to the Oracle's terms https://www.oracle.com/downloads/licenses/distribution-license.html (t|f)? [t] ");
9880           boolean oracleTermsAgreed = readFromStdInBoolean(true, "Placeholder");
9881           File oracleLibPath = new File(path+File.separator+"slashRoot"+File.separator+"opt"+
9882               File.separator+"grouper"+File.separator+"grouperWebapp"+File.separator+"WEB-INF"+File.separator+"lib");
9883           if (!oracleTermsAgreed) {
9884             System.out.print("Place the oracle jdbc jar in " + oracleLibPath.getAbsolutePath() + " and press <return> to continue ");
9885             readFromStdIn("Placeholder");
9886           } else {
9887             System.out.print("Do you want the installer to install the oracle jar (t|f)? [t] ");
9888             boolean shouldContinue = readFromStdInBoolean(true, "Placeholder");
9889             if (shouldContinue) {
9890               
9891               File oracleLibJarPath = new File(path+File.separator+"slashRoot"+File.separator+"opt"+
9892                   File.separator+"grouper"+File.separator+"grouperWebapp"+File.separator+"WEB-INF"+File.separator+"lib"+File.separator+"ojdbc8-19.3.0.0.jar");
9893               
9894               GrouperInstallerUtils.createParentDirectories(oracleLibJarPath);
9895               
9896               downloadFile("https://repo1.maven.org/maven2/com/oracle/ojdbc/ojdbc8/19.3.0.0/ojdbc8-19.3.0.0.jar", oracleLibJarPath.getAbsolutePath(), "");
9897             }
9898           }
9899           
9900           
9901         }
9902         
9903       }
9904       System.out.print("Database user: ");
9905       String newDbUser = readFromStdIn("grouperInstaller.autorun.dbUser");
9906       if (!GrouperInstallerUtils.isBlank(newDbUser)) {
9907         this.dbUser = newDbUser;
9908       }
9909       System.out.print("Database password (note, you aren't setting the pass here, you are using an existing pass, this will be echoed back) [" 
9910           + GrouperInstallerUtils.defaultIfEmpty(this.dbPass, "<blank>") + "]: ");
9911       String newDbPass = readFromStdIn("grouperInstaller.autorun.dbPass");
9912       
9913       String encryptedDbPassword =  "";
9914       
9915       if (GrouperInstallerUtils.isNotBlank(newDbPass)) {
9916         encryptedDbPassword = new Crypto(morphStringProperties.getProperty("encrypt.key") + "w").encrypt(newDbPass);
9917       }
9918 
9919       editPropertiesFile(grouperHibernatePropertiesFile, "hibernate.connection.url", this.dbUrl, false);
9920       editPropertiesFile(grouperHibernatePropertiesFile, "hibernate.connection.username", this.dbUser, false);
9921       editPropertiesFile(grouperHibernatePropertiesFile, "hibernate.connection.password", encryptedDbPassword, false);
9922       
9923       editPropertiesFile(grouperHibernatePropertiesFile, "grouper.is.ui.basicAuthn", "true", false);
9924       editPropertiesFile(grouperHibernatePropertiesFile, "grouper.is.ws.basicAuthn", "true", false);
9925     } 
9926     
9927     System.out.print("Do you want to init the database and auto-upgrade for subsequent containers of the same major and minor version of Grouper (t|f)? [t] ");
9928     boolean autoInitDatabase = readFromStdInBoolean(true, "Placeholder");
9929     boolean initDbDocker = false;
9930     if (autoInitDatabase) {
9931       String versionWithAnyPatch = dockerImageVersion.substring(0, dockerImageVersion.lastIndexOf("."));
9932       versionWithAnyPatch = versionWithAnyPatch + ".*";
9933       editPropertiesFile(grouperHibernatePropertiesFile, "registry.auto.ddl.upToVersion", versionWithAnyPatch, false);
9934       initDbDocker = true;
9935     } else {
9936       System.out.print("Do you want to init the database one time now (t|f)? [t] ");
9937       boolean initDatabaseOneTime = readFromStdInBoolean(true, "Placeholder");
9938       if (initDatabaseOneTime == true) {
9939         initDbDocker = true;
9940       }
9941     }
9942     
9943     contentToWrite = new StringBuilder();
9944     contentToWrite.append("Run the following command to init the database. It is not a required step.");
9945     contentToWrite.append("\n");
9946     
9947     StringBuilder buildInitCommand = new StringBuilder();
9948     buildInitCommand.append("docker run --detach ");
9949     buildInitCommand.append("--mount type=bind,src=");
9950     buildInitCommand.append(path+File.separator);
9951     buildInitCommand.append("logs,dst=/opt/grouper/logs ");
9952     buildInitCommand.append("--mount type=bind,src=");
9953     buildInitCommand.append(path+File.separator);
9954     buildInitCommand.append("slashRoot,dst=/opt/grouper/slashRoot ");
9955     buildInitCommand.append("--name gsh ");
9956     buildInitCommand.append("i2incommon/grouper:"+dockerImageVersion );
9957     buildInitCommand.append(" gsh -registry -check -runscript -noprompt" );
9958     
9959     contentToWrite.append(buildInitCommand.toString());
9960     contentToWrite.append("\n\n");
9961     contentToWrite.append("\n\n");
9962     
9963     try {      
9964       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
9965     } catch (Exception e) {
9966       System.out.println("Could not write to README.txt file.");
9967     }
9968     
9969     boolean removeDockerGshContainer = false;
9970     if (initDbDocker) {
9971       boolean dbInitialized = false;
9972       try {
9973         commands = new ArrayList<String>();
9974         commands.add(shCommand());
9975         commands.add("-c");
9976         
9977         commands.add(buildInitCommand.toString());
9978         
9979         CommandResult dockerDbInitCommandResult = GrouperInstallerUtils.execCommand(
9980             GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
9981             new File("."), null, false, false, true);
9982         
9983         if (dockerDbInitCommandResult.getExitCode() == 0) {
9984           dbInitialized = true;
9985         }
9986         
9987       } catch (Exception e) {}
9988       
9989       if (dbInitialized == false) {
9990         System.out.println("Could not initialize db. Run the following command manually in another terminal window/tab.");
9991         System.out.println(buildInitCommand.toString());
9992         System.out.print("Press <return> to continue once the command has been run: ");
9993         readFromStdIn("Placeholder");
9994       } else {
9995         removeDockerGshContainer = true;
9996         StringBuilder dockerPsCommand = new StringBuilder();
9997         dockerPsCommand.append("docker ps -a --filter \"name=gsh\" --format \"{{.Status}}\" ");
9998         
9999         for (int i=0; i<100; i++) {
10000           System.out.println("Waiting for docker command to finish.");
10001           GrouperInstallerUtils.sleep(4000);
10002           
10003           commands = new ArrayList<String>();
10004           commands.add(shCommand());
10005           commands.add("-c");
10006           
10007           commands.add(dockerPsCommand.toString());
10008           
10009           CommandResult dockerPsCommandResult = GrouperInstallerUtils.execCommand(
10010               GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
10011               new File("."), null, false, false, true);
10012           
10013           if (dockerPsCommandResult.getExitCode() == 0) {
10014             
10015             String output = dockerPsCommandResult.getOutputText();
10016             if (output.contains("Exited")) {
10017               
10018               GrouperInstallerUtils.sleep(20000);
10019               
10020               commands = new ArrayList<String>();
10021               commands.add(shCommand());
10022               commands.add("-c");
10023               
10024               StringBuilder dockerLogsCommand = new StringBuilder();
10025               dockerLogsCommand.append("docker logs gsh ");
10026               
10027               commands.add(dockerLogsCommand.toString());
10028               
10029               CommandResult dockerLogsCommandResult = GrouperInstallerUtils.execCommand(
10030                   GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
10031                   new File("."), null, false, false, true);
10032               
10033               if (dockerLogsCommandResult.getExitCode() == 0) {
10034                 String logs = dockerLogsCommandResult.getOutputText();
10035                 
10036                 File dbInitLogsFile = new File(path+File.separator+"docker_logs_init_db_"+new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss_SSS").format(new Date())+".log");
10037                 GrouperInstallerUtils.fileCreate(dbInitLogsFile);
10038                 
10039                 try {      
10040                   Files.write(Paths.get(dbInitLogsFile.getAbsolutePath()), logs.getBytes(), StandardOpenOption.APPEND);
10041                 } catch (Exception e) {
10042                   System.out.println("Could not write logs to "+dbInitLogsFile.getAbsolutePath()+" file. ");
10043                 }
10044                 
10045                 System.out.println("docker database initialization logs are at: "+dbInitLogsFile.getAbsolutePath());
10046                 
10047                 if (logs.contains("Script was executed successfully")
10048                     && logs.contains("SQL statements executed successfully")
10049                     && !logs.contains("Exception") && !logs.contains("exception")) {
10050                   System.out.println("From the logs: Script was executed successfully");
10051                   System.out.print("Press <return> to continue. ");
10052                   readFromStdIn("Placeholder");
10053                 } else {
10054                   System.out.println("Could not find success in docker logs. Look in the logs at the location above and make sure there are no exceptions. ");
10055                   System.out.print("Press <return> to continue. ");
10056                   readFromStdIn("Placeholder");
10057                 }
10058                 
10059                 try (BufferedReader reader = new BufferedReader(new FileReader(dbInitLogsFile))) {
10060                   StringBuilder logContent = new StringBuilder();
10061                   String line;
10062                   int lineNumber =0;
10063                   while( (line = reader.readLine()) != null) {
10064                     logContent.append(line);
10065                     logContent.append("\n");
10066                     lineNumber++;
10067                     
10068                     if (lineNumber > 24) {
10069                       break;
10070                     }
10071                   }
10072                   System.out.println("First 25 lines of logs are below: ");
10073                   System.out.println(logContent);
10074                 } catch (Exception e) {}
10075                 
10076               } else {
10077                 System.out.println("Could not retrieve logs for 'gsh' container. Please run 'docker logs gsh' manually and make sure there are no exceptions before proceeding. ");
10078                 System.out.print("Press <return> to continue. ");
10079                 readFromStdIn("Placeholder");
10080               }
10081               
10082               break;
10083             } 
10084             
10085           } else {
10086             System.out.println("Could not detect the status of the docker command. Please run '" +dockerPsCommand+ "' manually in another terminal window and once the status shows Exited, press <return> to continue. ");
10087             readFromStdIn("Placeholder");
10088             break;
10089           }
10090           
10091         }
10092         
10093       }
10094       
10095     }
10096     
10097     // remove gsh docker container
10098     contentToWrite = new StringBuilder();
10099     contentToWrite.append("Run 'docker rm -f gsh' to remove the gsh container.");
10100     contentToWrite.append("\n\n");
10101     contentToWrite.append("\n\n");
10102     try {
10103       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
10104     } catch (Exception e) {
10105       System.out.println("Could not write to README.txt file.");
10106     }
10107     
10108     if (removeDockerGshContainer) {
10109       commands = new ArrayList<String>();
10110       commands.add(shCommand());
10111       commands.add("-c");
10112       commands.add("docker rm -f gsh");
10113       
10114       boolean dockerRemovedGshContainer = false;
10115       String dockerRmCommandError = null;
10116       try {
10117         CommandResult dockerRmCommandResult = GrouperInstallerUtils.execCommand(
10118             GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
10119             new File("."), null, false, false, true);
10120         if (dockerRmCommandResult.getExitCode() == 0) {
10121           dockerRemovedGshContainer = true;
10122         } else if (GrouperInstallerUtils.isNotBlank(dockerRmCommandResult.getErrorText())) {
10123           dockerRmCommandError = dockerRmCommandResult.getErrorText();
10124         }
10125       } catch (Exception e) {
10126         dockerRemovedGshContainer = false;
10127       }
10128       
10129       if (dockerRemovedGshContainer) {
10130         System.out.println("Removed 'gsh' container successfully. ");
10131       } else if (dockerRmCommandError != null) {
10132         System.out.println("Could not remove docker gsh container. Please run 'docker rm -f gsh' in another terminal window before proceeding. ");
10133       }
10134       
10135       System.out.print("Press <return> to continue ");
10136       readFromStdIn("Placeholder");
10137     }
10138     
10139     // Add passwords for grouper ui
10140     contentToWrite = new StringBuilder();    
10141     contentToWrite.append("If you want to use grouper basic authentication for UI, follow the instructions below.");
10142     contentToWrite.append("\n");
10143     contentToWrite.append("Create createGrouperSystemPasswordUi.gsh file in "+path+File.separator+"slashRoot"+File.separator+"opt"+
10144         File.separator+"grouper"+File.separator+"grouperWebapp"+File.separator+"WEB-INF"+File.separator+"bin");
10145     contentToWrite.append("\n");
10146     contentToWrite.append("Add the following lines to createGrouperSystemPasswordUi.gsh. Replace placeholder with actual values below.");
10147     contentToWrite.append("\n");
10148     contentToWrite.append("GrouperSession grouperSession = GrouperSession.startRootSession();");
10149     contentToWrite.append("\n");
10150     contentToWrite.append("GrouperPasswordSave grouperPasswordSave = new GrouperPasswordSave();");
10151     contentToWrite.append("\n");
10152     contentToWrite.append("grouperPasswordSave.assignUsername(\"GrouperSystem\");");
10153     contentToWrite.append("\n");
10154     contentToWrite.append("grouperPasswordSave.assignEntityType(\"username\");");
10155     contentToWrite.append("\n");
10156     contentToWrite.append("grouperPasswordSave.assignPassword(\"<password>\");");
10157     contentToWrite.append("\n");
10158     contentToWrite.append("grouperPasswordSave.assignApplication(GrouperPassword.Application.UI);");
10159     contentToWrite.append("\n");
10160     contentToWrite.append("new Authentication().assignUserPassword(grouperPasswordSave);");
10161     contentToWrite.append("\n\n");
10162     contentToWrite.append("\n\n");
10163     
10164     try {
10165       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
10166     } catch (Exception e) {
10167       System.out.println("Could not write to README.txt file.");
10168     }
10169     
10170     //System.out.print("Do you want to use grouper authenitcation for UI? (t|f)? [t]: ");
10171     //boolean useGrouperAuthenticationUi = readFromStdInBoolean(true, "grouperInstaller.autorun.useBuiltInGrouperAuthenticationUI");
10172     boolean useGrouperAuthenticationUi = true;
10173     
10174     if (useGrouperAuthenticationUi) {
10175       System.out.print("Enter the password for user 'GrouperSystem' for grouper UI: ");
10176       String uiPassword = readFromStdIn("Placeholder");
10177       
10178       while (true) {
10179         if (uiPassword == null || uiPassword.trim().length() < 4 ) {
10180           System.out.print("Invalid password. Minimum 4 characters required. Please try again: ");
10181           uiPassword = readFromStdIn("Placeholder");
10182         } else {
10183           break;
10184         }
10185       }
10186       
10187       File grouperUiSystemPasswordSh = new File(path+File.separator+"slashRoot"+File.separator+"opt"+
10188           File.separator+"grouper"+File.separator+"grouperWebapp"+File.separator+"WEB-INF"+File.separator+"bin"+File.separator+"createGrouperSystemPasswordUi.gsh");
10189       GrouperInstallerUtils.createParentDirectories(grouperUiSystemPasswordSh);
10190       boolean reuseCreateGrouperSystemPasswordUiFile = false;
10191       while(true) {
10192         if (grouperUiSystemPasswordSh.exists()) {
10193           System.out.println("createGrouperSystemPasswordUi.gsh already exists at "+grouperUiSystemPasswordSh.getParent()+" ");
10194           System.out.print("Do you want to reuse it (t|f) [t]: ");
10195           reuseCreateGrouperSystemPasswordUiFile = readFromStdInBoolean(true, "Placeholder");
10196           if (reuseCreateGrouperSystemPasswordUiFile) {
10197             System.out.println("Going to reuse existing createGrouperSystemPasswordUi.gsh file. ");
10198             break;
10199           } else {
10200             System.out.print("Delete createGrouperSystemPasswordUi.sh and press <return> to continue ");
10201             readFromStdIn("nothing");
10202             grouperUiSystemPasswordSh = new File(path+File.separator+"slashRoot"+File.separator+"opt"+
10203                 File.separator+"grouper"+File.separator+"grouperWebapp"+File.separator+"WEB-INF"+File.separator+"bin"+File.separator+"createGrouperSystemPasswordUi.gsh");
10204             continue;
10205           }
10206         } else {
10207           GrouperInstallerUtils.fileCreate(grouperUiSystemPasswordSh);
10208           break;
10209         }
10210       }
10211       
10212       if (reuseCreateGrouperSystemPasswordUiFile == false) {
10213         StringBuilder createPasswordCommands = new StringBuilder();
10214         createPasswordCommands.append("GrouperSession grouperSession = GrouperSession.startRootSession(); \n");
10215         createPasswordCommands.append("GrouperPasswordSave grouperPasswordSave = new GrouperPasswordSave(); \n");
10216         createPasswordCommands.append("grouperPasswordSave.assignUsername(\"GrouperSystem\"); \n");
10217         createPasswordCommands.append("grouperPasswordSave.assignEntityType(\"username\"); \n");
10218         createPasswordCommands.append("grouperPasswordSave.assignPassword(\""+uiPassword+"\");");
10219         createPasswordCommands.append("\n");
10220         
10221         createPasswordCommands.append("grouperPasswordSave.assignApplication(GrouperPassword.Application.UI); \n");
10222         createPasswordCommands.append("new Authentication().assignUserPassword(grouperPasswordSave); \n");
10223         
10224         GrouperInstallerUtils.writeStringToFile(grouperUiSystemPasswordSh, createPasswordCommands.toString());
10225       }
10226       
10227       // run docker command to add UI password to grouper
10228       contentToWrite = new StringBuilder();
10229       contentToWrite.append("Run the following command to add UI password to grouper.");
10230       contentToWrite.append("\n");
10231       StringBuilder uiPasswordDockerCommand = new StringBuilder();
10232       uiPasswordDockerCommand.append("docker run --detach ");
10233       uiPasswordDockerCommand.append("--mount type=bind,src=");
10234       uiPasswordDockerCommand.append(path+File.separator);
10235       uiPasswordDockerCommand.append("logs,dst=/opt/grouper/logs ");
10236       uiPasswordDockerCommand.append("--mount type=bind,src=");
10237       uiPasswordDockerCommand.append(path+File.separator);
10238       uiPasswordDockerCommand.append("slashRoot,dst=/opt/grouper/slashRoot ");
10239       uiPasswordDockerCommand.append("--name gsh ");
10240       uiPasswordDockerCommand.append("i2incommon/grouper:"+dockerImageVersion );
10241       uiPasswordDockerCommand.append(" gsh /opt/grouper/grouperWebapp/WEB-INF/bin/createGrouperSystemPasswordUi.gsh");
10242       contentToWrite.append(uiPasswordDockerCommand.toString());
10243       contentToWrite.append("\n\n");
10244       contentToWrite.append("\n\n");
10245       try {
10246         Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
10247       } catch (Exception e) {
10248         System.out.println("Could not write to README.txt file.");
10249       }
10250       
10251       removeDockerGshContainer = false;
10252       boolean uiPasswordCreated = false;
10253       try {
10254         commands = new ArrayList<String>();
10255         commands.add(shCommand());
10256         commands.add("-c");
10257         commands.add(uiPasswordDockerCommand.toString());
10258         commandResult = GrouperInstallerUtils.execCommand(
10259             GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
10260             new File("."), null, false, false, true);
10261         
10262         if (commandResult.getExitCode() == 0) {
10263           uiPasswordCreated = true;
10264         }
10265         
10266       } catch (Exception e) {}
10267       
10268       if (uiPasswordCreated == false) {
10269         System.out.println("Could not create password for grouper UI. Run the following command manually. ");
10270         System.out.println(uiPasswordDockerCommand.toString());
10271         System.out.print("Press <return> to continue ");
10272         readFromStdIn("Placeholder");
10273       } else {
10274         
10275         removeDockerGshContainer = true;
10276         StringBuilder dockerPsCommand = new StringBuilder();
10277         dockerPsCommand.append("docker ps -a --filter \"name=gsh\" --format \"{{.Status}}\" ");
10278         
10279         for (int i=0; i<100; i++) {
10280           System.out.println("Waiting for docker command to finish.");
10281           GrouperInstallerUtils.sleep(4000);
10282           
10283           commands = new ArrayList<String>();
10284           commands.add(shCommand());
10285           commands.add("-c");
10286           
10287           commands.add(dockerPsCommand.toString());
10288           
10289           CommandResult dockerPsCommandResult = GrouperInstallerUtils.execCommand(
10290               GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
10291               new File("."), null, false, false, true);
10292           
10293           if (dockerPsCommandResult.getExitCode() == 0) {
10294             
10295             String output = dockerPsCommandResult.getOutputText();
10296             if (output.contains("Exited")) {
10297               
10298               GrouperInstallerUtils.sleep(20000);
10299               
10300               commands = new ArrayList<String>();
10301               commands.add(shCommand());
10302               commands.add("-c");
10303               
10304               StringBuilder dockerLogsCommand = new StringBuilder();
10305               dockerLogsCommand.append("docker logs gsh ");
10306               
10307               commands.add(dockerLogsCommand.toString());
10308               
10309               CommandResult dockerLogsCommandResult = GrouperInstallerUtils.execCommand(
10310                   GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
10311                   new File("."), null, false, false, true);
10312               
10313               if (dockerLogsCommandResult.getExitCode() == 0) {
10314                 String logs = dockerLogsCommandResult.getOutputText();
10315                 
10316                 File uiPasswordLogsFile = new File(path+File.separator+"docker_logs_ui_password_"+new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss_SSS").format(new Date())+".log");
10317                 GrouperInstallerUtils.fileCreate(uiPasswordLogsFile);
10318                 
10319                 try {      
10320                   Files.write(Paths.get(uiPasswordLogsFile.getAbsolutePath()), logs.getBytes(), StandardOpenOption.APPEND);
10321                 } catch (Exception e) {
10322                   System.out.println("Could not write logs to "+uiPasswordLogsFile.getAbsolutePath()+" file. ");
10323                 }
10324                 
10325                 System.out.println("docker ui password setup logs are at: "+uiPasswordLogsFile.getAbsolutePath());
10326                 
10327                 if (logs.contains("===> null\n" + 
10328                     "groovy:000> :exit") && !logs.contains("Exception") && !logs.contains("exception")) {
10329                   System.out.println("Password was created successfully.");
10330                   System.out.print("Press <return> to continue. ");
10331                   readFromStdIn("Placeholder");
10332                 } else {
10333                   System.out.println("Could not find success in docker logs. Look in the logs at the location above and make sure there are no exceptions. ");
10334                   System.out.print("Press <return> to continue. ");
10335                   readFromStdIn("Placeholder");
10336                 }
10337                 
10338                 try (BufferedReader reader = new BufferedReader(new FileReader(uiPasswordLogsFile))) {
10339                   StringBuilder logContent = new StringBuilder();
10340                   String line;
10341                   int lineNumber =0;
10342                   while( (line = reader.readLine()) != null) {
10343                     logContent.append(line);
10344                     logContent.append("\n");
10345                     lineNumber++;
10346                     
10347                     if (lineNumber > 24) {
10348                       break;
10349                     }
10350                   }
10351                   System.out.println("First 25 lines of logs are below: ");
10352                   System.out.println(logContent);
10353                 } catch (Exception e) {}
10354                 
10355               } else {
10356                 System.out.println("Could not retrieve logs for 'gsh' container. Please run 'docker logs gsh' manually and make sure there are no exceptions before proceeding. ");
10357                 System.out.print("Press <return> to continue. ");
10358                 readFromStdIn("Placeholder");
10359               }
10360               
10361               break;
10362             } 
10363             
10364           } else {
10365             System.out.println("Could not detect the status of the docker command. Please run '" +dockerPsCommand+ "' manually in another terminal window and once the status shows Exited, press <return> to continue. ");
10366             readFromStdIn("Placeholder");
10367             break;
10368           }
10369           
10370         }
10371         
10372       }
10373       
10374       if (removeDockerGshContainer) {
10375         commands = new ArrayList<String>();
10376         commands.add(shCommand());
10377         commands.add("-c");
10378         commands.add("docker rm -f gsh");
10379         
10380         boolean dockerRemovedGshContainer = false;
10381         String dockerRmCommandError = null;
10382         try {
10383           CommandResult dockerRmCommandResult = GrouperInstallerUtils.execCommand(
10384               GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
10385               new File("."), null, false, false, true);
10386           if (dockerRmCommandResult.getExitCode() == 0) {
10387             dockerRemovedGshContainer = true;
10388           } else if (GrouperInstallerUtils.isNotBlank(dockerRmCommandResult.getErrorText())) {
10389             dockerRmCommandError = dockerRmCommandResult.getErrorText();
10390           }
10391         } catch (Exception e) {
10392           dockerRemovedGshContainer = false;
10393         }
10394         
10395         if (dockerRemovedGshContainer) {
10396           System.out.println("Removed gsh container successfully. ");
10397         } else if (dockerRmCommandError != null) {
10398           System.out.println("Could not remove docker gsh container. Please run 'docker rm -f gsh' in another terminal window before proceeding. ");
10399         }
10400         
10401         System.out.print("Press <return> to continue ");
10402         readFromStdIn("Placeholder");
10403       }
10404         
10405       // remove createGrouperSystemPasswordUi.sh file
10406       contentToWrite = new StringBuilder();    
10407       contentToWrite.append("Delete createGrouperSystemPasswordUi.gsh file from "+path+File.separator+"slashRoot"+File.separator+"opt"+
10408           File.separator+"grouper"+File.separator+"grouperWebapp"+File.separator+"WEB-INF"+File.separator+"bin");
10409       contentToWrite.append(" because it contains password in plain text.");
10410       contentToWrite.append("\n\n");
10411       try {
10412         Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
10413       } catch (Exception e) {
10414         System.out.println("Could not write to README.txt file.");
10415       }
10416       
10417       grouperUiSystemPasswordSh.delete();
10418       
10419     }
10420     
10421     // Add passwords for grouper ws
10422     contentToWrite = new StringBuilder();    
10423     contentToWrite.append("If you want to use grouper basic authentication for grouper web services, follow the instructions below.");
10424     contentToWrite.append("\n");
10425     contentToWrite.append("Create createGrouperSystemPasswordWs.gsh file in "+path+File.separator+"slashRoot"+File.separator+"opt"+
10426         File.separator+"grouper"+File.separator+"grouperWebapp"+File.separator+"WEB-INF"+File.separator+"bin");
10427     contentToWrite.append("\n");
10428     contentToWrite.append("Add the following lines to createGrouperSystemPasswordWs.gsh. Replace placeholder with actual values below.");
10429     contentToWrite.append("\n");
10430     contentToWrite.append("GrouperSession grouperSession = GrouperSession.startRootSession();");
10431     contentToWrite.append("\n");
10432     contentToWrite.append("GrouperPasswordSave grouperPasswordSave = new GrouperPasswordSave();");
10433     contentToWrite.append("\n");
10434     contentToWrite.append("grouperPasswordSave.assignUsername(\"GrouperSystem\");");
10435     contentToWrite.append("\n");
10436     contentToWrite.append("grouperPasswordSave.assignEntityType(\"username\");");
10437     contentToWrite.append("\n");
10438     contentToWrite.append("grouperPasswordSave.assignPassword(\"<password>\");");
10439     contentToWrite.append("\n");
10440     contentToWrite.append("grouperPasswordSave.assignApplication(GrouperPassword.Application.WS);");
10441     contentToWrite.append("\n");
10442     contentToWrite.append("new Authentication().assignUserPassword(grouperPasswordSave);");
10443     contentToWrite.append("\n\n");
10444     contentToWrite.append("\n\n");
10445     
10446     try {
10447       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
10448     } catch (Exception e) {
10449       System.out.println("Could not write to README.txt file.");
10450     }
10451 //    System.out.print("Do you want to use grouper authenitcation for WS? (t|f)? [t]: ");
10452 //    boolean useGrouperAuthenticationWs = readFromStdInBoolean(true, "grouperInstaller.autorun.useBuiltInGrouperAuthenticationWS");
10453     boolean useGrouperAuthenticationWs = true;
10454     
10455     if (useGrouperAuthenticationWs) {
10456       System.out.print("Please enter the password for user 'GrouperSystem' for grouper web services: ");
10457       String wsPassword = readFromStdIn("Placeholder");
10458       
10459       while (true) {
10460         if (wsPassword == null || wsPassword.trim().length() < 4 ) {
10461           System.out.print("Invalid password. Minimum 4 characters required. Please try again: ");
10462           wsPassword = readFromStdIn("Placeholder");
10463         } else {
10464           break;
10465         }
10466       }
10467       
10468       File grouperWsSystemPasswordSh = new File(path+File.separator+"slashRoot"+File.separator+"opt"+
10469           File.separator+"grouper"+File.separator+"grouperWebapp"+File.separator+"WEB-INF"+File.separator+"bin"+File.separator+"createGrouperSystemPasswordWs.gsh");
10470       GrouperInstallerUtils.createParentDirectories(grouperWsSystemPasswordSh);
10471       boolean reuseCreateGrouperSystemPasswordWsFile = false;
10472       while(true) {
10473         if (grouperWsSystemPasswordSh.exists()) {
10474           System.out.println("createGrouperSystemPasswordWs.gsh already exists at "+grouperWsSystemPasswordSh.getParent()+" ");
10475           System.out.print("Do you want to reuse it (t|f) [t]: ");
10476           reuseCreateGrouperSystemPasswordWsFile = readFromStdInBoolean(true, "Placeholder");
10477           if (reuseCreateGrouperSystemPasswordWsFile) {
10478             System.out.println("Going to reuse existing createGrouperSystemPasswordWs.gsh file. ");
10479             break;
10480           } else {
10481             System.out.print("Delete createGrouperSystemPasswordWs.gsh and press <return> to continue ");
10482             readFromStdIn("nothing");
10483             grouperWsSystemPasswordSh = new File(path+File.separator+"slashRoot"+File.separator+"opt"+
10484                 File.separator+"grouper"+File.separator+"grouperWebapp"+File.separator+"WEB-INF"+File.separator+"bin"+File.separator+"createGrouperSystemPasswordWs.gsh");
10485             continue;
10486           }
10487         } else {
10488           GrouperInstallerUtils.fileCreate(grouperWsSystemPasswordSh);
10489           break;
10490         }
10491       }
10492       
10493       if (reuseCreateGrouperSystemPasswordWsFile == false) {
10494         StringBuilder createPasswordCommands = new StringBuilder();
10495         createPasswordCommands.append("GrouperSession grouperSession = GrouperSession.startRootSession(); \n");
10496         createPasswordCommands.append("GrouperPasswordSave grouperPasswordSave = new GrouperPasswordSave(); \n");
10497         createPasswordCommands.append("grouperPasswordSave.assignUsername(\"GrouperSystem\"); \n");
10498         createPasswordCommands.append("grouperPasswordSave.assignEntityType(\"username\"); \n");
10499         createPasswordCommands.append("grouperPasswordSave.assignPassword(\""+wsPassword+"\"); \n");
10500         createPasswordCommands.append("\n");
10501         
10502         createPasswordCommands.append("grouperPasswordSave.assignApplication(GrouperPassword.Application.WS); \n");
10503         createPasswordCommands.append("new Authentication().assignUserPassword(grouperPasswordSave); \n");
10504         
10505         GrouperInstallerUtils.writeStringToFile(grouperWsSystemPasswordSh, createPasswordCommands.toString());
10506       }
10507       
10508    // run docker command to add WS password to grouper
10509       contentToWrite = new StringBuilder();
10510       contentToWrite.append("Run the following command to add WS password to grouper.");
10511       contentToWrite.append("\n");
10512       StringBuilder wsPasswordDockerCommand = new StringBuilder();
10513       wsPasswordDockerCommand.append("docker run --detach ");
10514       wsPasswordDockerCommand.append("--mount type=bind,src=");
10515       wsPasswordDockerCommand.append(path+File.separator);
10516       wsPasswordDockerCommand.append("logs,dst=/opt/grouper/logs ");
10517       wsPasswordDockerCommand.append("--mount type=bind,src=");
10518       wsPasswordDockerCommand.append(path+File.separator);
10519       wsPasswordDockerCommand.append("slashRoot,dst=/opt/grouper/slashRoot ");
10520       wsPasswordDockerCommand.append("--name gsh ");
10521       wsPasswordDockerCommand.append("i2incommon/grouper:"+dockerImageVersion );
10522       wsPasswordDockerCommand.append(" gsh /opt/grouper/grouperWebapp/WEB-INF/bin/createGrouperSystemPasswordWs.gsh");
10523       contentToWrite.append(wsPasswordDockerCommand.toString());
10524       contentToWrite.append("\n\n");
10525       contentToWrite.append("\n\n");
10526       try {
10527         Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
10528       } catch (Exception e) {
10529         System.out.println("Could not write to README.txt file.");
10530       }
10531       
10532       boolean wsPasswordCreated = false;
10533       removeDockerGshContainer = false;
10534       try {
10535         commands = new ArrayList<String>();
10536         commands.add(shCommand());
10537         commands.add("-c");
10538         
10539         commands.add(wsPasswordDockerCommand.toString());
10540         
10541         commandResult = GrouperInstallerUtils.execCommand(
10542             GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
10543             new File("."), null, false, false, true);
10544         
10545         if (commandResult.getExitCode() == 0) {
10546           wsPasswordCreated = true;
10547         }
10548         
10549       } catch (Exception e) {}
10550       
10551       if (wsPasswordCreated == false) {
10552         System.out.println("Could not create password for WS. Run the following command manually.");
10553         System.out.println(wsPasswordDockerCommand.toString());
10554         System.out.print("Press <return> to continue");
10555         readFromStdIn("Placeholder");
10556       } else {
10557 
10558         removeDockerGshContainer = true;
10559         StringBuilder dockerPsCommand = new StringBuilder();
10560         dockerPsCommand.append("docker ps -a --filter \"name=gsh\" --format \"{{.Status}}\" ");
10561         
10562         for (int i=0; i<100; i++) {
10563           System.out.println("Waiting for docker command to finish.");
10564           GrouperInstallerUtils.sleep(4000);
10565           
10566           commands = new ArrayList<String>();
10567           commands.add(shCommand());
10568           commands.add("-c");
10569           
10570           commands.add(dockerPsCommand.toString());
10571           
10572           CommandResult dockerPsCommandResult = GrouperInstallerUtils.execCommand(
10573               GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
10574               new File("."), null, false, false, true);
10575           
10576           if (dockerPsCommandResult.getExitCode() == 0) {
10577             
10578             String output = dockerPsCommandResult.getOutputText();
10579             if (output.contains("Exited")) {
10580               
10581               GrouperInstallerUtils.sleep(20000);
10582               
10583               commands = new ArrayList<String>();
10584               commands.add(shCommand());
10585               commands.add("-c");
10586               
10587               StringBuilder dockerLogsCommand = new StringBuilder();
10588               dockerLogsCommand.append("docker logs gsh ");
10589               
10590               commands.add(dockerLogsCommand.toString());
10591               
10592               CommandResult dockerLogsCommandResult = GrouperInstallerUtils.execCommand(
10593                   GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
10594                   new File("."), null, false, false, true);
10595               
10596               if (dockerLogsCommandResult.getExitCode() == 0) {
10597                 String logs = dockerLogsCommandResult.getOutputText();
10598                 
10599                 File wsPasswordLogsFile = new File(path+File.separator+"docker_logs_ws_password_"+new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss_SSS").format(new Date())+".log");
10600                 GrouperInstallerUtils.fileCreate(wsPasswordLogsFile);
10601                 
10602                 try {      
10603                   Files.write(Paths.get(wsPasswordLogsFile.getAbsolutePath()), logs.getBytes(), StandardOpenOption.APPEND);
10604                 } catch (Exception e) {
10605                   System.out.println("Could not write logs to "+wsPasswordLogsFile.getAbsolutePath()+" file. ");
10606                 }
10607                 
10608                 System.out.println("docker ws password setup logs are at: "+wsPasswordLogsFile.getAbsolutePath());
10609                 
10610                 if (logs.contains("===> null\n" + 
10611                     "groovy:000> :exit") && !logs.contains("Exception") && !logs.contains("exception")) {
10612                   System.out.println("Password was created successfully.");
10613                   System.out.print("Press <return> to continue. ");
10614                   readFromStdIn("Placeholder");
10615                 } else {
10616                   System.out.println("Could not find success in docker logs. Look in the logs at the location above and make sure there are no exceptions. ");
10617                   System.out.print("Press <return> to continue. ");
10618                   readFromStdIn("Placeholder");
10619                 }
10620                 
10621                 try (BufferedReader reader = new BufferedReader(new FileReader(wsPasswordLogsFile))) {
10622                   StringBuilder logContent = new StringBuilder();
10623                   String line;
10624                   int lineNumber = 0;
10625                   while( (line = reader.readLine()) != null) {
10626                     logContent.append(line);
10627                     logContent.append("\n");
10628                     lineNumber++;
10629                     
10630                     if (lineNumber > 24) {
10631                       break;
10632                     }
10633                   }
10634                   System.out.println("First 25 lines of logs are below: ");
10635                   System.out.println(logContent);
10636                 } catch (Exception e) {}
10637                 
10638               } else {
10639                 System.out.println("Could not retrieve logs for 'gsh' container. Please run 'docker logs gsh' manually and make sure there are no exceptions before proceeding. ");
10640                 System.out.print("Press <return> to continue. ");
10641                 readFromStdIn("Placeholder");
10642               }
10643               
10644               break;
10645             } 
10646             
10647           } else {
10648             System.out.println("Could not detect the status of the docker command. Please run '" +dockerPsCommand+ "' manually in another terminal window and once the status shows Exited, press <return> to continue. ");
10649             readFromStdIn("Placeholder");
10650             break;
10651           }
10652           
10653         }
10654       
10655       }
10656       
10657       if (removeDockerGshContainer) {
10658         commands = new ArrayList<String>();
10659         commands.add(shCommand());
10660         commands.add("-c");
10661         commands.add("docker rm -f gsh");
10662         
10663         boolean dockerRemovedGshContainer = false;
10664         String dockerRmCommandError = null;
10665         try {
10666           CommandResult dockerRmCommandResult = GrouperInstallerUtils.execCommand(
10667               GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
10668               new File("."), null, false, false, true);
10669           if (dockerRmCommandResult.getExitCode() == 0) {
10670             dockerRemovedGshContainer = true;
10671           } else if (GrouperInstallerUtils.isNotBlank(dockerRmCommandResult.getErrorText())) {
10672             dockerRmCommandError = dockerRmCommandResult.getErrorText();
10673           }
10674         } catch (Exception e) {
10675           dockerRemovedGshContainer = false;
10676         }
10677         
10678         if (dockerRemovedGshContainer) {
10679           System.out.println("Removed 'gsh' container successfully. ");
10680         } else if (dockerRmCommandError != null) {
10681           System.out.println("Could not remove docker gsh container. Please run 'docker rm -f gsh' in another terminal window before proceeding. ");
10682         }
10683         
10684         System.out.print("Press <return> to continue ");
10685         readFromStdIn("Placeholder");
10686       }
10687       
10688       // remove createGrouperSystemPasswordWs.sh file
10689       contentToWrite = new StringBuilder();    
10690       contentToWrite.append("Delete createGrouperSystemPasswordWs.gsh file from "+path+File.separator+"slashRoot"+File.separator+"opt"+
10691           File.separator+"grouper"+File.separator+"grouperWebapp"+File.separator+"WEB-INF"+File.separator+"bin");
10692       contentToWrite.append(" because it contains password in plain text.");
10693       contentToWrite.append("\n\n");
10694       try {
10695         Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
10696       } catch (Exception e) {
10697         System.out.println("Could not write to README.txt file.");
10698       }
10699       
10700       grouperWsSystemPasswordSh.delete();
10701       
10702     }
10703     
10704     Integer portNumberInt = 8080;
10705     System.out.print("Please provide the host HTTP (not HTTPS) port number to run the UI container (default 8080): ");
10706     while (true) {
10707       String portNumber = readFromStdIn("Placeholder");
10708       if (GrouperInstallerUtils.isNotBlank(portNumber)) {
10709         try {
10710           portNumberInt = Integer.valueOf(portNumber);
10711           break;
10712         } catch (Exception e) {
10713           System.out.print("Invalid port number '"+portNumber+"'. Please try again. (default 8080): ");
10714         }
10715       } else {
10716         break;
10717       }
10718     }
10719     
10720     // start the container
10721     contentToWrite = new StringBuilder();
10722     contentToWrite.append("Run the following command to start the container.");
10723     contentToWrite.append("\n");
10724     StringBuilder grouperContainerStartDockerCommand = new StringBuilder();
10725     grouperContainerStartDockerCommand.append("docker run");
10726     //grouperContainerStartDockerCommand.append(" -e GROUPER_UI='true' -e GROUPER_WS='true' -e GROUPER_DAEMON='true' ");
10727     //grouperContainerStartDockerCommand.append("-e RUN_APACHE='true' -e RUN_SHIB_SP='false' -e RUN_TOMEE='true' ");
10728     grouperContainerStartDockerCommand.append(" --detach --publish "+portNumberInt.toString()+":8080 ");
10729     grouperContainerStartDockerCommand.append("--mount type=bind,src=");
10730     grouperContainerStartDockerCommand.append(path+File.separator);
10731     grouperContainerStartDockerCommand.append("logs,dst=/opt/grouper/logs ");
10732     grouperContainerStartDockerCommand.append("--mount type=bind,src=");
10733     grouperContainerStartDockerCommand.append(path+File.separator);
10734     grouperContainerStartDockerCommand.append("slashRoot,dst=/opt/grouper/slashRoot ");
10735     grouperContainerStartDockerCommand.append("--restart always --name grouper-ui ");
10736     grouperContainerStartDockerCommand.append("i2incommon/grouper:"+dockerImageVersion );
10737     grouperContainerStartDockerCommand.append(" ui" );
10738     contentToWrite.append(grouperContainerStartDockerCommand.toString());
10739     contentToWrite.append("\n\n");
10740     contentToWrite.append("\n\n");
10741     try {
10742       Files.write(Paths.get(readmeFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
10743     } catch (Exception e) {
10744       System.out.println("Could not write to README.txt file.");
10745     }
10746     
10747     
10748     System.out.print("Press <return> to start the UI container: ");
10749     readFromStdIn("Placeholder");
10750     boolean dockerGrouperContainerStarted = false;
10751     try {
10752       commands = new ArrayList<String>();
10753       commands.add(shCommand());
10754       commands.add("-c");
10755       
10756       commands.add(grouperContainerStartDockerCommand.toString());
10757       
10758       commandResult = GrouperInstallerUtils.execCommand(
10759           GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
10760           new File("."), null, false, false, true);
10761       
10762       if (commandResult.getExitCode() == 0) {
10763         dockerGrouperContainerStarted = true;
10764       }
10765       
10766     } catch (Exception e) {}
10767     
10768     if (dockerGrouperContainerStarted == false) {
10769       System.out.println("Could not start grouper container. Run the following command manually.");
10770       System.out.println(grouperContainerStartDockerCommand.toString());
10771     } else {
10772       System.out.println("Inside container grouper runs on port 8080");
10773     }
10774     
10775     System.out.println("Logs are at: "+new File(path+File.separator+"logs").getAbsolutePath());
10776     System.out.println("Command history and documentation are in "+ readmeFile.getAbsolutePath());
10777     System.out.println("Grouper UI is running at : http://localhost:"+portNumberInt+"/grouper/");
10778     System.out.print("Press <return> to exit ");
10779     readFromStdIn("Placeholder");
10780   }
10781     
10782   /**
10783    * 
10784    */
10785   private void mainBuildContainerLogic() {
10786     
10787     //Find out the working directory.  This ends in a file separator
10788     this.grouperTarballDirectoryString = grouperUpgradeTempDirectory();
10789     
10790     this.version = GrouperInstallerUtils.propertiesValue("grouper.version", false);
10791     
10792     if (GrouperInstallerUtils.isBlank(this.version)) {
10793       this.version = getClass().getPackage().getImplementationVersion();
10794     }
10795     
10796     System.out.println("Installing grouper version: " + this.version);
10797     
10798     downloadAndUnzipMaven();
10799     
10800     //####################################
10801     //download apache tomcat
10802     File tomcatDir = downloadTomcat();
10803     File unzippedTomcatFile = unzip(tomcatDir.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc");
10804     this.untarredTomcatDir = untar(unzippedTomcatFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc", null);
10805     
10806     // download grouper tag from github
10807     File grouperSourceCodeDir = downloadGrouperSourceTagFromGithub();
10808     File unzippedGrouperSourceCodeFile = unzip(grouperSourceCodeDir.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc");
10809     File untarredGrouperSourceCodeDir = untar(unzippedGrouperSourceCodeFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc", null);
10810     
10811     
10812     String grouperUntarredReleaseDir = untarredGrouperSourceCodeDir.getAbsolutePath().substring(0, untarredGrouperSourceCodeDir.getAbsolutePath().lastIndexOf(File.separator));
10813     grouperUntarredReleaseDir = grouperUntarredReleaseDir + File.separator + "grouper-" + untarredGrouperSourceCodeDir.getName() ;
10814     
10815     // now create an output directory (webapp) and tomcat
10816     String containerDirString = grouperContainerDirectory();
10817     File containerTomcatDir = new File(containerDirString + "tomcat");
10818     containerTomcatDir.mkdirs();
10819     
10820     File webAppDir = new File(containerDirString + "webapp");
10821     webAppDir.mkdirs();
10822         
10823     // let's start with grouper-ui/webapp directory. We will copy everything else later
10824     GrouperInstallerUtils.copyDirectory(new File(grouperUntarredReleaseDir + File.separator + "grouper-ui" + File.separator+"webapp"), webAppDir);
10825     
10826     File webInfDir = new File(webAppDir+File.separator+"WEB-INF");
10827     webInfDir.mkdirs(); // should already be there
10828     File webInfConfDir = new File(webInfDir+File.separator+"conf");
10829     webInfConfDir.mkdirs();
10830     
10831     File libDir = new File(webInfDir+File.separator+"lib");
10832     libDir.mkdirs();
10833     
10834     File libUiAndDaemonDir = new File(webInfDir+File.separator+"libUiAndDaemon");
10835     libUiAndDaemonDir.mkdirs();
10836     
10837     File libWsDir = new File(webInfDir+File.separator+"libWs");
10838     libWsDir.mkdirs();
10839     
10840     File modulesDir = new File(webInfDir+File.separator+"modules");
10841     modulesDir.mkdirs();
10842     File servicesDir = new File(webInfDir+File.separator+"services");
10843     servicesDir.mkdirs();
10844     File classesDir = new File(webInfDir+File.separator+"classes");
10845     classesDir.mkdirs();
10846     File binDir = new File(webInfDir+File.separator+"bin");
10847     binDir.mkdirs();
10848     
10849     // go in grouper-container and run mvn dependency:copy-dependencies
10850     Map<File, File> projectDirToOutputLibDir = new LinkedHashMap<File, File>();
10851     projectDirToOutputLibDir.put(new File(grouperUntarredReleaseDir + File.separator + "grouper-container" + File.separator + "grouper-api-container"), libDir);
10852     projectDirToOutputLibDir.put(new File(grouperUntarredReleaseDir + File.separator + "grouper-container" + File.separator + "grouper-uiDaemon-container"), libUiAndDaemonDir);
10853     projectDirToOutputLibDir.put(new File(grouperUntarredReleaseDir + File.separator + "grouper-container" + File.separator + "grouper-ws-container"), libWsDir);
10854     
10855     List<String> commands = new ArrayList<String>();
10856     addMavenCommands(commands);
10857     
10858     commands.add("-DincludeScope=runtime");
10859     commands.add("-Dgrouper.version="+this.version);
10860  
10861     commands.add("dependency:copy-dependencies");
10862           
10863     for (File file: projectDirToOutputLibDir.keySet()) {
10864       System.out.println("\n##################################");
10865       System.out.println("Downloading third party jars for "+ file.getName()+" with command:\n" 
10866           + convertCommandsIntoCommand(commands) + "\n");
10867       
10868       CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
10869           true, true, null, new File(file.getAbsolutePath()), null, true);
10870       
10871       if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
10872         System.out.println("stderr: " + commandResult.getErrorText());
10873       }
10874       if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
10875         System.out.println("stdout: " + commandResult.getOutputText());
10876       }
10877     }
10878     
10879     // now copy all dependency jars into corresponding container/webapp/WEB-INF/lib{""|UiAndDaemon|Ws|}
10880     try {
10881       Set<String> allGrouperApiJars = new HashSet<String>();
10882       for (File file: projectDirToOutputLibDir.keySet()) {        
10883         File jarsDirectory = new File(file.getAbsolutePath()+File.separator+"target"+File.separator+"dependency");
10884         
10885         if (allGrouperApiJars.size() == 0) { // we are relying on the first item in projectDirToOutputLibDir being the grouper api 
10886           for (File jarFile: findAllLibraryFiles(jarsDirectory.getAbsolutePath())) {
10887             allGrouperApiJars.add(jarFile.getName());
10888           }
10889           // copy the grouper api jars into container/webapp/WEB-INF/lib
10890           GrouperInstallerUtils.copyDirectory(jarsDirectory, projectDirToOutputLibDir.get(file), null, true);
10891           continue;
10892         }
10893         
10894         // make sure the jar file is already not there
10895         for (File jarFile: findAllLibraryFiles(jarsDirectory.getAbsolutePath())) {
10896           if (allGrouperApiJars.contains(jarFile.getName()) == false) {
10897             File destFile = new File(projectDirToOutputLibDir.get(file).getAbsolutePath() + File.separator + jarFile.getName());
10898             GrouperInstallerUtils.copyFile(jarFile, destFile);
10899           }
10900         }
10901         
10902       }
10903     } catch (Exception e) {
10904       throw new RuntimeException("Could not copy jars from dependency directories ", e);
10905     }
10906     
10907     // delete all grouper snapshots jars from WEB-INF/lib
10908 //    List<File> allLibraryJars = findAllLibraryFiles(libDir.getAbsolutePath());
10909 //    
10910 //    for (File file: allLibraryJars) {
10911 //      if (file.getName().contains("grouper") && file.getName().contains("SNAPSHOT")) {
10912 //        GrouperInstallerUtils.fileDelete(file);
10913 //      }
10914 //    }
10915     
10916     // now copy grouper/conf, grouper-ws/conf, grouper-ui/conf and grouperClient/conf to classesDir
10917     
10918     List<File> projectsToGetConfFrom = new ArrayList<File>();
10919     projectsToGetConfFrom.add(new File(grouperUntarredReleaseDir + File.separator + "grouper"));
10920     projectsToGetConfFrom.add(new File(grouperUntarredReleaseDir + File.separator + "grouper-ws"+File.separator+"grouper-ws"));
10921     projectsToGetConfFrom.add(new File(grouperUntarredReleaseDir + File.separator + "grouper-ui"));
10922     projectsToGetConfFrom.add(new File(grouperUntarredReleaseDir + File.separator + "grouper-misc"+File.separator+"grouperClient"));
10923     
10924     try {
10925       for (File file: projectsToGetConfFrom) {
10926         File confDir = new File(file.getAbsolutePath()+File.separator+"conf");
10927         if (confDir.exists()) {
10928           GrouperInstallerUtils.copyDirectory(confDir, classesDir, null, true);
10929         }
10930       }
10931     } catch (Exception e) {
10932       throw new RuntimeException("Could not copy files from conf directory to classes directory", e);
10933     }
10934     
10935     // now copy all misc/*.example.properties into classes directory and rename them to not have example in the filename
10936     
10937     File miscDir = new File(grouperUntarredReleaseDir + File.separator + "grouper" + File.separator + "misc");
10938     File[] filesToBeCopied = miscDir.listFiles(new FilenameFilter() {
10939       
10940       @Override
10941       public boolean accept(File dir, String name) {
10942         return name.endsWith("example.properties") && !name.equals("grouper.text.en.us.example.properties");
10943       }
10944     });
10945     
10946     for (File miscFileToBeCopied: filesToBeCopied) {
10947       String newFileName = miscFileToBeCopied.getName().replace(".example", "");
10948       File destFile = new File(classesDir.getAbsolutePath() + File.separator + newFileName);
10949       GrouperInstallerUtils.copyFile(miscFileToBeCopied, destFile);
10950     }
10951     
10952     // now copy grouper.text.en.us.example.properties into classes/grouperText
10953     File textFileToBeCopied = new File(grouperUntarredReleaseDir + File.separator + "grouper" + 
10954         File.separator + "misc" + File.separator + "grouper.text.en.us.example.properties");
10955     
10956     File textDestFile = new File(classesDir.getAbsolutePath() + File.separator + "grouperText" + 
10957         File.separator + "grouper.text.en.us.properties");
10958     GrouperInstallerUtils.copyFile(textFileToBeCopied, textDestFile);
10959     
10960     
10961     // now copy all grouper-ws/webapp specific files into outputDir/webapp
10962     File grouperWsWebinfDir = new File(grouperUntarredReleaseDir+File.separator+"grouper-ws"+File.separator+
10963         "grouper-ws"+File.separator+"webapp"+File.separator+"WEB-INF");
10964     
10965     GrouperInstallerUtils.copyDirectory(new File(grouperWsWebinfDir.getAbsolutePath()+File.separator+"modules"), modulesDir);
10966     GrouperInstallerUtils.copyDirectory(new File(grouperWsWebinfDir.getAbsolutePath()+File.separator+"services"), servicesDir);
10967     GrouperInstallerUtils.copyDirectory(new File(grouperWsWebinfDir.getAbsolutePath()+File.separator+"conf"), webInfConfDir);
10968     
10969     // now copy grouper/bin contents into outputDir/webapp/WEB-INF/bin
10970     GrouperInstallerUtils.copyDirectory(new File(grouperUntarredReleaseDir + File.separator + "grouper" + File.separator + "bin"), binDir);
10971     
10972     // make gsh.sh executable
10973     commands = GrouperInstallerUtils.toList("chmod", "+x", binDir.getAbsolutePath() + File.separator + "gsh.sh");
10974 
10975     System.out.println("Making sure gsh.sh is executable with command: " + convertCommandsIntoCommand(commands) + "\n");
10976 
10977     CommandResult commandResult = GrouperInstallerUtils.execCommand(
10978         GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
10979         binDir, null, true);
10980     
10981     if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
10982       System.out.println("stderr: " + commandResult.getErrorText());
10983     }
10984     if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
10985       System.out.println("stdout: " + commandResult.getOutputText());
10986     }
10987     
10988     // now download all the grouper project jars
10989     downloadGrouperJarsIntoLibDirectory(webInfDir);
10990     
10991     // delete slf4j related jars from all the lib dirs
10992     deleteJarsFromLibDirs(webInfDir);
10993     
10994     // take care of conflicting jars
10995     reportOnConflictingJars(libDir.getAbsolutePath());
10996     reportOnConflictingJars(libUiAndDaemonDir.getAbsolutePath());
10997     reportOnConflictingJars(libWsDir.getAbsolutePath());
10998     
10999     // copy apache-tomcat-x.y.z to tomcat
11000     // why can't uncompressed directory has the same name??? :((
11001     File tomcatUntarredDir = new File(this.grouperTarballDirectoryString + File.separator + "apache-tomcat-" + this.tomcatVersion());
11002     try {      
11003       GrouperInstallerUtils.copyDirectory(tomcatUntarredDir, containerTomcatDir, null, true);
11004     } catch (Exception e) {
11005       throw new RuntimeException("Could not copy untarred tomcat into container/tomcat", e);
11006     }
11007     
11008     // put logging related jars in tomcat/bin directory
11009     File tomcatBinDir = new File(containerTomcatDir + File.separator + "bin");
11010     
11011 //    downloadFile("https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.13.1/log4j-core-2.13.1.jar", tomeeBinDir.getAbsolutePath() + File.separator + "log4j-core-2.13.1.jar", "");
11012 //    downloadFile("https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-jul/2.13.1/log4j-jul-2.13.1.jar", tomeeBinDir.getAbsolutePath() + File.separator + "log4j-jul-2.13.1.jar", "");
11013 //    downloadFile("https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.13.1/log4j-api-2.13.1.jar", tomeeBinDir.getAbsolutePath() + File.separator + "log4j-api-2.13.1.jar", "");
11014 //    
11015 //    // put slf4j in lib dir
11016 //    downloadFile("https://repo1.maven.org/maven2/org/slf4j/slf4j-log4j12/1.7.21/slf4j-log4j12-1.7.21.jar", libDir + File.separator + "slf4j-log4j12-1.7.21.jar", "");
11017     
11018     // point tomcat to downloaded webpp
11019     configureTomcatGrouperUberWebapp(containerTomcatDir, webAppDir);
11020     
11021     // copy slf4j from tomee/lib to web-inf/lib
11022 //    File tomeeLibDir = new File(containerTomeeDir + File.separator + "lib");
11023 //    GrouperInstallerUtils.copyFile(new File(tomeeLibDir.getAbsolutePath() + File.separator + "slf4j-jdk14-1.7.21.jar"), new File(libDir.getAbsolutePath() + File.separator + "slf4j-jdk14-1.7.21.jar"));
11024 //    GrouperInstallerUtils.copyFile(new File(tomeeLibDir.getAbsolutePath() + File.separator + "slf4j-api-1.7.21.jar"), new File(libDir.getAbsolutePath() + File.separator + "slf4j-api-1.7.21.jar"));
11025     
11026   }
11027   
11028   /**
11029    * 
11030    */
11031   //TODO delete everything related to install
11032   @Deprecated
11033   private void mainInstallLogic() {
11034     
11035     //####################################
11036     //Find out what directory to install to.  This ends in a file separator
11037     this.grouperInstallDirectoryString = grouperInstallDirectory();
11038 
11039     //Find out what directory to upgrade to.  This ends in a file separator
11040     this.grouperTarballDirectoryString = grouperUpgradeTempDirectory();
11041 
11042     //####################################
11043     //get default ip address
11044     System.out.print("Enter the default IP address for checking ports (just hit enter to accept the default unless on a machine with no network, might want to change to 127.0.0.1): [0.0.0.0]: ");
11045     this.defaultIpAddress = readFromStdIn("grouperInstaller.autorun.defaultIpAddressForPorts");
11046     
11047     if (GrouperInstallerUtils.isBlank(this.defaultIpAddress)) {
11048       this.defaultIpAddress = "0.0.0.0";
11049     }
11050 
11051     if (!GrouperInstallerUtils.equals("0.0.0.0", this.defaultIpAddress)) {
11052       System.out.println("Note, you will probably need to change the tomcat server.xml IP addresses...");
11053     }
11054     
11055     //####################################
11056     //System.out.println("Grouper install directory is: " + grouperInstallDirectoryFile.getAbsolutePath());
11057 
11058     this.version = GrouperInstallerUtils.propertiesValue("grouper.version", true);
11059     System.out.println("Installing grouper version: " + this.version);
11060     //see if it is already downloaded
11061     
11062     //download api and set executable and dos2unix etc
11063     downloadAndConfigureApi();
11064 
11065     //####################################
11066     //ask about database
11067 
11068     File localGrouperHibernatePropertiesFile = new File(this.untarredApiDir.getAbsoluteFile() + File.separator + "conf" 
11069         + File.separator + "grouper.hibernate.properties");
11070 
11071     Properties grouperHibernateProperties = GrouperInstallerUtils.propertiesFromFile(localGrouperHibernatePropertiesFile);
11072 
11073     this.dbUrl = GrouperInstallerUtils.defaultString(grouperHibernateProperties.getProperty("hibernate.connection.url"));
11074     this.dbUser = GrouperInstallerUtils.defaultString(grouperHibernateProperties.getProperty("hibernate.connection.username"));
11075     this.dbPass = GrouperInstallerUtils.defaultString(grouperHibernateProperties.getProperty("hibernate.connection.password"));
11076 
11077     System.out.println("\n##################################\n");
11078     System.out.println("Example mysql URL: jdbc:mysql://localhost:3306/grouper");
11079     System.out.println("Example oracle URL: jdbc:oracle:thin:@server.school.edu:1521:sid");
11080     System.out.println("Example postgres URL: jdbc:postgresql://localhost:5432/database");
11081     System.out.println("Example mssql URL: jdbc:sqlserver://localhost:3280;databaseName=grouper");
11082     System.out.print("\nEnter the database URL [" + this.dbUrl + "]: ");
11083     String newDbUrl = readFromStdIn("grouperInstaller.autorun.dbUrl");
11084     if (!GrouperInstallerUtils.isBlank(newDbUrl)) {
11085       this.dbUrl = newDbUrl;
11086       if (newDbUrl.contains("postgresql") || newDbUrl.contains("sqlserver")) {
11087         System.out.println("Note: you need to change the search sql in the jdbc source in the grouperApi/conf/sources.xml... the change is in the comments in that file");
11088         for (int i=0;i<3;i++) {
11089           System.out.print("Ready to continue? (t|f)? [t] ");
11090           boolean shouldContinue = readFromStdInBoolean(true, "grouperInstaller.autorun.dbContinueAfterChangeSourcesXmlForPostgresSqlServer");
11091           if (shouldContinue) {
11092             break;
11093           }
11094         }
11095       }
11096     }
11097     System.out.print("Database user [" + this.dbUser + "]: ");
11098     String newDbUser = readFromStdIn("grouperInstaller.autorun.dbUser");
11099     if (!GrouperInstallerUtils.isBlank(newDbUser)) {
11100       this.dbUser = newDbUser;
11101     }
11102     System.out.print("Database password (note, you aren't setting the pass here, you are using an existing pass, this will be echoed back) [" 
11103         + GrouperInstallerUtils.defaultIfEmpty(this.dbPass, "<blank>") + "]: ");
11104     String newDbPass = readFromStdIn("grouperInstaller.autorun.dbPass");
11105     if (!GrouperInstallerUtils.isBlank(newDbPass)) {
11106       this.dbPass = newDbPass;
11107     }
11108 
11109     this.giDbUtils = new GiDbUtils(this.dbUrl, this.dbUser, this.dbPass);
11110     this.giDbUtils.registerDriverOnce(this.grouperInstallDirectoryString);
11111 
11112     //####################################
11113     //change the config file
11114     //get the config file
11115 
11116     //lets edit the three properties:
11117     System.out.println("Editing " + localGrouperHibernatePropertiesFile.getAbsolutePath() + ": ");
11118     editPropertiesFile(localGrouperHibernatePropertiesFile, "hibernate.connection.url", this.dbUrl, false);
11119     editPropertiesFile(localGrouperHibernatePropertiesFile, "hibernate.connection.username", this.dbUser, false);
11120     editPropertiesFile(localGrouperHibernatePropertiesFile, "hibernate.connection.password", this.dbPass, false);
11121 
11122     //####################################
11123     //check to see if listening on port?
11124 
11125     //#####################################
11126     //add driver to classpath
11127     //note, we are note really doing this now, we are using drivers already on classpath since this doesnt work
11128     //this.addDriverJarToClasspath();
11129 
11130     //####################################
11131     //check connection to database
11132     checkDatabaseConnection();
11133     
11134     //####################################
11135     // patch the API
11136     this.upgradeExistingApplicationDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.untarredApiDir.getAbsolutePath());
11137     this.upgradeExistingClassesDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.untarredApiDir.getAbsolutePath())
11138         + "conf" + File.separator;
11139     this.upgradeExistingLibDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.untarredApiDir.getAbsolutePath())
11140         + "lib" + File.separator;
11141     this.upgradeExistingBinDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.untarredApiDir.getAbsolutePath())
11142             + "bin" + File.separator;
11143     patchApi();
11144 
11145     //make sure log4j is debugging sql statements
11146     log4jDebugSql(this.upgradeExistingClassesDirectoryString + "log4j.properties");
11147     
11148     //####################################
11149     //ask then init the DB
11150     initDb();
11151     addQuickstartSubjects();
11152     addQuickstartData();
11153     
11154     //####################################
11155     //download and configure ui
11156     System.out.print("Do you want to install the user interface (t|f)? [t]: ");
11157     boolean installUi = readFromStdInBoolean(true, "grouperInstaller.autorun.installUi");
11158     if (installUi) {
11159       downloadAndConfigureUi();
11160     }
11161     
11162     //####################################
11163     //get ant
11164     downloadAndUnzipAnt();
11165     
11166     //####################################
11167     //look for or ask or download tomcat
11168     File tomcatDir = downloadTomcat();
11169     File unzippedTomcatFile = unzip(tomcatDir.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc");
11170     this.untarredTomcatDir = untar(unzippedTomcatFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc", 
11171         new File(this.grouperInstallDirectoryString));
11172 
11173     //####################################
11174     //ask for tomcat port
11175     configureTomcat();
11176 
11177     File apiPatchStatusFile = new File(GrouperInstallerUtils.fileAddLastSlashIfNotExists(
11178         this.untarredApiDir.getAbsolutePath()) + "grouperPatchStatus.properties");
11179 
11180     //####################################
11181     //build UI
11182     if (installUi) {
11183 
11184       buildUi(true);
11185 
11186       //####################################
11187       //configureTomcatUiWebapp
11188       configureTomcatUiWebapp();
11189   
11190       //####################################
11191       //copy api patch level to ui
11192       File uiPatchStatusFile = new File(GrouperInstallerUtils.fileAddLastSlashIfNotExists(
11193           this.grouperUiBuildToDirName()) + "WEB-INF" + File.separator + "grouperPatchStatus.properties");
11194       System.out.println("Copying applied API patch status to UI:");
11195       System.out.println("  - from: "  + apiPatchStatusFile.getAbsolutePath());
11196       System.out.println("  - to: "  + uiPatchStatusFile.getAbsolutePath());
11197       GrouperInstallerMergePatchFiles.mergePatchFiles(
11198           apiPatchStatusFile, uiPatchStatusFile, true);
11199 
11200 
11201       //####################################
11202       // patch the ui
11203       this.upgradeExistingApplicationDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.grouperUiBuildToDirName());
11204       this.upgradeExistingClassesDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.grouperUiBuildToDirName())
11205           + "WEB-INF" + File.separator + "classes" + File.separator ;
11206       this.upgradeExistingLibDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.grouperUiBuildToDirName())
11207           + "WEB-INF" + File.separator + "lib" + File.separator;
11208       this.upgradeExistingBinDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.grouperUiBuildToDirName())
11209           + "WEB-INF" + File.separator + "bin" + File.separator ;
11210 
11211       //####################################
11212       // temp fix for 2.4 full install - the api and ui /bin dirs differ in 2.4; this step
11213       // will sync by renaming ui files that differ and then copy the api versions
11214       String apiBinSource = GrouperInstallerUtils.fileAddLastSlashIfNotExists(
11215               this.untarredApiDir.getAbsolutePath()) + "bin" + File.separator;
11216       String targetBinSouce = GrouperInstallerUtils.fileAddLastSlashIfNotExists(
11217               this.grouperUiBuildToDirName()) + "WEB-INF" + File.separator + "bin" + File.separator;
11218       String[] filesToCopyFromApiBin = new String[]{"gsh.sh", "gsh.bat", "gsh", "README.txt", "setenv.example.bat", "setenv.example.sh"};
11219       this.grouperBaseBakDir = this.grouperTarballDirectoryString + "bak_UI_"
11220               + new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss_SSS").format(new Date()) + File.separator;
11221 
11222       System.out.println("Reconciling differences between API and UI /bin directories...");
11223       syncFilesInDirWithBackup(apiBinSource, targetBinSouce, filesToCopyFromApiBin);
11224       this.grouperBaseBakDir = null;
11225 
11226       this.patchUi();
11227     }
11228     
11229     //####################################
11230     //set the GrouperSystem password
11231     tomcatConfigureGrouperSystem();
11232 
11233     if (installUi) {
11234       //####################################
11235       //bounce tomcat
11236       tomcatBounce("restart");
11237       
11238       //####################################
11239       //tell user to go to url
11240       System.out.println("##################################\n");
11241       System.out.println("Go here for the Grouper UI (change hostname if on different host): http://localhost:" + this.tomcatHttpPort + "/" + this.tomcatUiPath + "/");
11242       System.out.println("\n##################################\n");
11243     }
11244     
11245     System.out.print("Do you want to install web services (t|f)? [t]: ");
11246     boolean installWs = readFromStdInBoolean(true, "grouperInstaller.autorun.installWs");
11247     
11248     if (installWs) {
11249       this.downloadAndUntarWs();
11250       
11251       //#################################### 
11252       //configure Ws
11253       this.configureWs();
11254       
11255       //####################################
11256       //build WS
11257       buildWs(true);
11258       
11259       //####################################
11260       //copy to tomcat
11261       configureTomcatWsWebapp();
11262   
11263       //####################################
11264       //copy api patch level to ui
11265       File wsPatchStatusFile = new File(GrouperInstallerUtils.fileAddLastSlashIfNotExists(
11266           this.grouperWsBuildToDirName()) + "WEB-INF" + File.separator + "grouperPatchStatus.properties");
11267       System.out.println("Copying applied API patch status to WS:");
11268       System.out.println("  - from: "  + apiPatchStatusFile.getAbsolutePath());
11269       System.out.println("  - to: "  + wsPatchStatusFile.getAbsolutePath());
11270       GrouperInstallerMergePatchFiles.mergePatchFiles(
11271           apiPatchStatusFile, wsPatchStatusFile, true);
11272 
11273       //####################################
11274       // patch the ws
11275       this.upgradeExistingApplicationDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.grouperWsBuildToDirName());
11276       this.upgradeExistingClassesDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.grouperWsBuildToDirName())
11277           + "WEB-INF" + File.separator + "classes" + File.separator ;
11278       this.upgradeExistingLibDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.grouperWsBuildToDirName())
11279           + "WEB-INF" + File.separator + "lib" + File.separator;
11280       this.upgradeExistingBinDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.grouperWsBuildToDirName())
11281               + "WEB-INF" + File.separator + "bin" + File.separator ;
11282 
11283       //####################################
11284       // temp fix for 2.4 full install - the api and ws /bin dirs differ in 2.4; this step
11285       // will sync by renaming ws files that differ and then copy the api versions
11286       String apiBinSource = GrouperInstallerUtils.fileAddLastSlashIfNotExists(
11287               this.untarredApiDir.getAbsolutePath()) + "bin" + File.separator;
11288       String targetBinSouce = GrouperInstallerUtils.fileAddLastSlashIfNotExists(
11289               this.grouperWsBuildToDirName()) + "WEB-INF" + File.separator + "bin" + File.separator;
11290       String[] filesToCopyFromApiBin = new String[]{"gsh.sh", "gsh.bat", "gsh", "README.txt", "setenv.example.bat", "setenv.example.sh"};
11291       this.grouperBaseBakDir = this.grouperTarballDirectoryString + "bak_WS_"
11292               + new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss_SSS").format(new Date()) + File.separator;
11293 
11294       System.out.println("Reconciling differences between API and WS /bin directories...");
11295       syncFilesInDirWithBackup(apiBinSource, targetBinSouce, filesToCopyFromApiBin);
11296       this.grouperBaseBakDir = null;
11297 
11298       this.patchWs();
11299   
11300       //####################################
11301       //bounce tomcat
11302       tomcatBounce("restart");
11303   
11304       //####################################
11305       //tell user to go to url
11306       System.out.println("This is the Grouper WS URL (change hostname if on different host): http://localhost:" + this.tomcatHttpPort + "/" + this.tomcatWsPath + "/");
11307     }
11308     
11309     System.out.print("Do you want to install the web services client (t|f)? [t]: ");
11310     boolean installClient = readFromStdInBoolean(true, "grouperInstaller.autorun.installClient");
11311     
11312     if (installClient) {
11313       //download and build client
11314       this.downloadAndBuildClient();
11315   
11316       //####################################
11317       //configure where WS is
11318       this.configureClient();
11319       
11320       if (installWs) {
11321         //####################################
11322         //add grouper system to WS group
11323         this.addGrouperSystemWsGroup();
11324         
11325         //####################################
11326         //run a client command
11327         this.runClientCommand();
11328       }
11329     }
11330 
11331     //####################################
11332     //install pspng
11333     System.out.print("Do you want to install the provisioning service provider next generation (t|f)? [t]: ");
11334     boolean installPspng = readFromStdInBoolean(true, "grouperInstaller.autorun.installPspng");
11335     if (installPspng) {
11336       downloadAndBuildPspng();  
11337       
11338       //copy jars
11339       GrouperInstallerUtils.copyDirectory(new File(this.untarredPspngDir.getAbsolutePath() + File.separator + "lib" + File.separator + "custom"), 
11340           new File(this.untarredApiDir.getAbsolutePath() + File.separator + "lib" + File.separator + "custom"));
11341       GrouperInstallerUtils.copyDirectory(new File(this.untarredPspngDir.getAbsolutePath() + File.separator + "dist"), 
11342           new File(this.untarredApiDir.getAbsolutePath() + File.separator + "lib" + File.separator + "custom"));
11343 
11344       //####################################
11345       // patch the PSP
11346       this.upgradeExistingApplicationDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.untarredApiDir.getAbsolutePath());
11347       this.upgradeExistingClassesDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.untarredApiDir.getAbsolutePath())
11348           + "conf" + File.separator;
11349       this.upgradeExistingLibDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.untarredApiDir.getAbsolutePath())
11350           + "lib" + File.separator;
11351       patchPspng();
11352             
11353     }
11354 
11355 
11356     if (!installPspng) {
11357       //####################################
11358       //install psp
11359       System.out.print("Do you want to install the provisioning service provider (t|f)? [t]: ");
11360       if (readFromStdInBoolean(true, "grouperInstaller.autorun.installPsp")) {
11361         downloadAndBuildPsp();              
11362         GrouperInstallerUtils.copyDirectory(this.untarredPspDir, this.untarredApiDir);
11363   
11364         //####################################
11365         // patch the PSP
11366         this.upgradeExistingApplicationDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.untarredApiDir.getAbsolutePath());
11367         this.upgradeExistingClassesDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.untarredApiDir.getAbsolutePath())
11368             + "conf" + File.separator;
11369         this.upgradeExistingLibDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.untarredApiDir.getAbsolutePath())
11370             + "lib" + File.separator;
11371         patchPsp();
11372               
11373       }
11374     }
11375     
11376     reportOnConflictingJars(this.upgradeExistingApplicationDirectoryString);
11377 
11378     //#####################################
11379     //start the loader
11380     startLoader(true);
11381     
11382     //#####################################
11383     //success
11384     System.out.println("\n##################################\n");
11385 
11386     System.out.println("\nInstallation success!");
11387 
11388 
11389     System.out.println("\nRun the installer's 'admin' function to get information and manage about your installation (db, tomcat, logs, etc)");
11390     
11391     if (installUi) {
11392       System.out.println("\nGo here for the Grouper UI (change hostname if on different host): http://localhost:" + this.tomcatHttpPort + "/" + this.tomcatUiPath + "/");
11393       
11394     }
11395     if (installWs) {
11396       System.out.println("\nThis is the Grouper WS URL (change hostname if on different host): http://localhost:" + this.tomcatHttpPort + "/" + this.tomcatWsPath + "/");
11397     }
11398     System.out.println("\n##################################\n");
11399 
11400   }
11401 
11402 
11403   /**
11404    * 
11405    */
11406   private void downloadAndBuildPsp() {
11407     File pspDir = downloadPsp();
11408     File unzippedPspFile = unzip(pspDir.getAbsolutePath(), "grouperInstaller.autorun.useLocalPspDownloadTarEtc");
11409     this.untarredPspDir = untar(unzippedPspFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalPspDownloadTarEtc", 
11410         null);
11411 
11412   }
11413   
11414   /**
11415    * 
11416    */
11417   private void downloadAndBuildPspng() {
11418     File pspngDir = downloadPspng();
11419     File unzippedPspngFile = unzip(pspngDir.getAbsolutePath(), "grouperInstaller.autorun.useLocalPspngDownloadTarEtc");
11420     this.untarredPspngDir = untar(unzippedPspngFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalPspngDownloadTarEtc", 
11421         null);
11422 
11423   }
11424   
11425   /** untarred psp dir file */
11426   private File untarredPspDir;
11427   
11428   /** untarred pspng dir file */
11429   private File untarredPspngDir;
11430   
11431   /**
11432    * 
11433    */
11434   public void downloadAndUnzipAnt() {
11435     File antDir = downloadAnt();
11436     File unzippedAntFile = unzip(antDir.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc");
11437     this.untarredAntDir = untar(unzippedAntFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc", null);
11438   }
11439 
11440   /**
11441    * 
11442    */
11443   public void downloadAndUnzipMaven() {
11444     File mavenDir = downloadMaven();
11445     File unzippedMavenFile = unzip(mavenDir.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc");
11446     this.untarredMavenDir = untar(unzippedMavenFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc", null);
11447   }
11448 
11449   /**
11450    * 
11451    */
11452   public void downloadAndUntarWs() {
11453 
11454     //####################################
11455     //download the ws
11456     File wsDir = downloadWs();
11457 
11458     //####################################
11459     //unzip/untar the ws file
11460     File unzippedWsFile = unzip(wsDir.getAbsolutePath(), "grouperInstaller.autorun.useLocalWsDownloadTarEtc");
11461     System.out.println("Unzipped Ws file is "+unzippedWsFile);
11462     this.untarredWsDir = untar(unzippedWsFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalWsDownloadTarEtc", 
11463         new File(this.grouperInstallDirectoryString));
11464 
11465   }
11466 
11467   /**
11468    * 
11469    */
11470   public void downloadAndConfigureUi() {
11471     //####################################
11472     //get UI
11473     File uiDir = downloadUi();
11474     
11475     //####################################
11476     //unzip/untar the ui file
11477     File unzippedUiFile = unzip(uiDir.getAbsolutePath(), "grouperInstaller.autorun.useLocalUiDownloadTarEtc");
11478     this.untarredUiDir = untar(unzippedUiFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalUiDownloadTarEtc", 
11479         new File(this.grouperInstallDirectoryString));
11480 
11481     //####################################
11482     //configure UI
11483     configureUi();
11484   }
11485 
11486   /**
11487    * 
11488    */
11489   public void downloadAndConfigureApi() {
11490     File apiFile = downloadApi();
11491     
11492     //####################################
11493     //unzip/untar the api file
11494     
11495     File unzippedApiFile = unzip(apiFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalApiDownloadTarEtc");
11496     File theUntarredApiDir = untar(unzippedApiFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalApiDownloadTarEtc", 
11497         new File(this.grouperInstallDirectoryString));
11498     
11499     File theGrouperJar = new File(GrouperInstallerUtils.fileAddLastSlashIfNotExists(theUntarredApiDir.getAbsolutePath())
11500         + "dist" + File.separator + "lib" + File.separator + "grouper.jar");
11501     
11502     gshExcutableAndDos2Unix(theUntarredApiDir.getAbsolutePath() + File.separator + "bin" + File.separator);
11503 
11504     //these might be set from UI or WS
11505     if (this.untarredApiDir == null) {
11506       this.untarredApiDir = theUntarredApiDir;
11507     }
11508     
11509     if (this.grouperJar == null) {
11510       this.grouperJar = theGrouperJar;
11511     }
11512   }
11513 
11514   /**
11515    * @param binDirLocation which includes trailing slash
11516    */
11517   public void gshExcutableAndDos2Unix(String binDirLocation) {
11518     gshExcutableAndDos2Unix(binDirLocation, null);
11519   }
11520 
11521   /**
11522    * run dos2unix on a file
11523    * @param file
11524    * @param fileNameInPrompt 
11525    * @param configSuffixAutorun 
11526    */
11527   public static void dos2unix(File file, String fileNameInPrompt, String configSuffixAutorun) {
11528     dos2unix(GrouperInstallerUtils.toSet(file), fileNameInPrompt, configSuffixAutorun);
11529   }
11530 
11531   /**
11532    * run dos2unix on a file
11533    * @param files
11534    * @param fileNameInPrompt e.g. gsh.sh
11535    * @param configSuffixAutorun suffix after grouperInstaller.autorun.dos2unix in properties file
11536    */
11537   public static void dos2unix(Collection<File> files, String fileNameInPrompt, String configSuffixAutorun) {
11538 
11539     if (!GrouperInstallerUtils.isWindows()) {
11540 
11541       System.out.print("Do you want to run dos2unix on " + fileNameInPrompt + " (t|f)? [t]: ");
11542       boolean dos2unixRunOnFile = readFromStdInBoolean(true, "grouperInstaller.autorun.dos2unix" + configSuffixAutorun);
11543       
11544       if (dos2unixRunOnFile) {
11545 
11546         for (File file : files) {
11547           
11548           if (!file.exists()) {
11549             continue;
11550           }
11551           
11552           List<String> commands = GrouperInstallerUtils.toList("dos2unix", 
11553               file.getAbsolutePath());
11554     
11555           System.out.println("Making sure " + file.getName() + " is in unix format: " + convertCommandsIntoCommand(commands) + "\n");
11556           String error = null;
11557           CommandResult commandResult = null;
11558           boolean didntWork = false;
11559           Throwable throwable = null;
11560           try {
11561             commandResult = GrouperInstallerUtils.execCommand(
11562                 GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
11563                 file.getParentFile(), null, false, true, false);
11564 
11565             if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
11566               System.out.println("stderr: " + commandResult.getErrorText());
11567             }
11568             if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
11569               System.out.println("stdout: " + commandResult.getOutputText());
11570             }
11571             continue;
11572           } catch (Throwable t) {
11573             didntWork = true;
11574             error = t.getMessage();
11575             throwable = t;
11576           }
11577           
11578           if (didntWork) {
11579             try {
11580               //lets try the java way?
11581               String fileContents = GrouperInstallerUtils.readFileIntoString(file);
11582               if (fileContents.contains("\r\n")) {
11583                 System.out.println("Problem with command 'dos2unix'.   Is it installed?  Converting to unix via java replacing \\r\\n with \\n: " + file.getAbsolutePath());
11584                 fileContents = fileContents.replaceAll("\r\n", "\n");
11585                 GrouperInstallerUtils.saveStringIntoFile(file, fileContents);
11586               }
11587               continue;
11588             } catch (Throwable t) {
11589               t.printStackTrace();
11590             }
11591           }
11592           
11593           if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
11594             System.out.println("stderr: " + commandResult.getErrorText());
11595           }
11596           if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
11597             System.out.println("stdout: " + commandResult.getOutputText());
11598           }
11599           if (!GrouperInstallerUtils.isBlank(error)) {
11600             if (throwable != null) {
11601               throwable.printStackTrace();
11602             }
11603             System.out.println("Error: " + error);
11604             System.out.println("NOTE: you might need to run this to convert newline characters to mac/unix:\n\n" +
11605                 "cat " + file.getAbsolutePath()
11606                 + " | col -b > " + file.getAbsolutePath() + "\n");
11607           }
11608         }
11609       }
11610 
11611     }
11612   }
11613   
11614   /**
11615    * @param binDirLocation which includes trailing slash
11616    * @param specify if specifying location
11617    */
11618   public void gshExcutableAndDos2Unix(String binDirLocation, String specify) {
11619     //lts make sure gsh is executable and in unix format
11620 
11621     if (!GrouperInstallerUtils.isWindows()) {
11622 
11623       specify = GrouperInstallerUtils.trimToEmpty(specify);
11624       
11625       if (specify.length() > 0) {
11626         specify += " ";
11627       }
11628       
11629       System.out.print("Do you want to set " + specify + "gsh script to executable (t|f)? [t]: ");
11630       boolean setGshFile = readFromStdInBoolean(true, "grouperInstaller.autorun.setGshScriptsToExecutable");
11631       
11632       if (setGshFile) {
11633       
11634         binDirLocation = GrouperInstallerUtils.fileAddLastSlashIfNotExists(binDirLocation);
11635         
11636         List<String> commands = GrouperInstallerUtils.toList("chmod", "+x", 
11637             binDirLocation + "gsh.sh");
11638   
11639         System.out.println("Making sure gsh.sh is executable with command: " + convertCommandsIntoCommand(commands) + "\n");
11640   
11641         CommandResult commandResult = GrouperInstallerUtils.execCommand(
11642             GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
11643             new File(binDirLocation), null, true);
11644         
11645         if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
11646           System.out.println("stderr: " + commandResult.getErrorText());
11647         }
11648         if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
11649           System.out.println("stdout: " + commandResult.getOutputText());
11650         }
11651 
11652         if (new File(binDirLocation + "gsh").exists()) {
11653           commands = GrouperInstallerUtils.toList("chmod", "+x", 
11654               binDirLocation + "gsh");
11655     
11656           System.out.println("Making sure gsh is executable with command: " + convertCommandsIntoCommand(commands) + "\n");
11657     
11658           commandResult = GrouperInstallerUtils.execCommand(
11659               GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
11660               new File(binDirLocation), null, true);
11661           
11662           if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
11663             System.out.println("stderr: " + commandResult.getErrorText());
11664           }
11665           if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
11666             System.out.println("stdout: " + commandResult.getOutputText());
11667           }
11668         }
11669         
11670         dos2unix(GrouperInstallerUtils.toSet(new File(binDirLocation + "gsh.sh"), new File(binDirLocation + "gsh")), "gsh.sh", "OnGsh");
11671 
11672       }
11673       
11674     }
11675   }
11676   
11677   /**
11678    * if we are debugging sql in log4j
11679    */
11680   private Boolean log4jDebugSql = null;
11681   
11682   /**
11683    * if this file has been taken care of for a while
11684    */
11685   private Set<File> log4jDebugDone = new HashSet<File>();
11686   
11687   /**
11688    * if this file has been taken care of for a while
11689    */
11690   private Set<File> removeLegacyHibernatePropertiesDone = new HashSet<File>();
11691   
11692   /**
11693    * @param hibernateFileLocation
11694    */
11695   public void removeLegacyHibernateProperties(String hibernateFileLocation) {
11696 
11697     //if not a file dont worry about it
11698     File hibernateFile = new File(hibernateFileLocation);
11699 
11700     if (this.removeLegacyHibernatePropertiesDone.contains(hibernateFile)) {
11701       return;
11702     }
11703 
11704     this.removeLegacyHibernatePropertiesDone.add(hibernateFile);
11705 
11706     if (!hibernateFile.exists()) {
11707       System.out.println("Cant find grouper.hibernate.properties: " + hibernateFileLocation);
11708       return;
11709     }
11710     
11711     //see if its there
11712     Properties hibernateProperties = GrouperInstallerUtils.propertiesFromFile(hibernateFile);
11713     String current = GrouperInstallerUtils.propertiesValue(hibernateProperties, "hibernate.cache.region.factory_class");
11714     
11715     if (current == null) {
11716       //not there, we're good
11717       return;
11718     }
11719 
11720 
11721     removeRedundantProperties(hibernateFile, GrouperInstallerUtils.toSet("hibernate.cache.region.factory_class"));
11722     System.out.println("File " + hibernateFile.getAbsolutePath() + " has property hibernate.cache.region.factory_class set to \"" + current + "\".  Removing since this is now in the grouper.hibernate.base.properties file.");
11723   }
11724   
11725   /**
11726    * @param log4jLocation
11727    */
11728   public void log4jDebugSql(String log4jLocation) {
11729 
11730     //if not a file dont worry about it
11731     File log4jFile = new File(log4jLocation);
11732 
11733     if (this.log4jDebugDone.contains(log4jFile)) {
11734       return;
11735     }
11736 
11737     this.log4jDebugDone.add(log4jFile);
11738 
11739     if (!log4jFile.exists()) {
11740       System.out.println("Cant find log4j.properties: " + log4jLocation);
11741       return;
11742     }
11743     
11744     //see if its already there
11745     Properties log4jProperties = GrouperInstallerUtils.propertiesFromFile(log4jFile);
11746     String currentAntEntry = GrouperInstallerUtils.propertiesValue(log4jProperties, "log4j.logger.org.apache.tools.ant");
11747     
11748     if (GrouperInstallerUtils.equalsIgnoreCase(GrouperInstallerUtils.trimToEmpty(currentAntEntry), "DEBUG")
11749         || GrouperInstallerUtils.equalsIgnoreCase(GrouperInstallerUtils.trimToEmpty(currentAntEntry), "INFO")
11750         || GrouperInstallerUtils.equalsIgnoreCase(GrouperInstallerUtils.trimToEmpty(currentAntEntry), "WARN")) {
11751       //we are already there
11752       return;
11753     }
11754 
11755     if (this.log4jDebugSql == null) {
11756       System.out.print("Do you want add log4j.logger.org.apache.tools.ant = WARN to " + log4jFile.getAbsolutePath() + " (recommended so you can see progress of SQL scripts) (t|f)? [t]: ");
11757       this.log4jDebugSql = readFromStdInBoolean(true, "grouperInstaller.autorun.log4jDebugSql");
11758     }
11759 
11760     if (this.log4jDebugSql) {
11761 
11762       editPropertiesFile(log4jFile, "log4j.logger.org.apache.tools.ant", "WARN", false);
11763 
11764     }
11765   }
11766 
11767   /**
11768    * 
11769    */
11770   public void downloadAndBuildClient() {
11771     //####################################
11772     //download the client
11773     File clientDir = downloadClient();
11774 
11775     //####################################
11776     //unzip/untar the client file
11777     File unzippedClientFile = unzip(clientDir.getAbsolutePath(), "grouperInstaller.autorun.useLocalClientDownloadTarEtc");
11778     this.untarredClientDir = untar(unzippedClientFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalClientDownloadTarEtc", 
11779         new File(this.grouperInstallDirectoryString));
11780     
11781   }
11782 
11783   /**
11784    * 
11785    */
11786   private int tomcatHttpPort = -1;
11787   
11788   /**
11789    * 
11790    */
11791   private int tomeeHttpPort = -1;
11792   
11793   
11794   /**
11795    * 
11796    */
11797   private void configureTomcat() {
11798     
11799     System.out.print("Do you want to set the tomcat memory limit (t|f)? [t]: ");
11800     boolean setTomcatMemory = readFromStdInBoolean(true, "grouperInstaller.autorun.setTomcatMemoryLimit");
11801     
11802     if (setTomcatMemory) {
11803       
11804       {
11805         File catalinaBatFile = new File(this.untarredTomcatDir.getAbsolutePath() + File.separator + "bin" + File.separator + "catalina.bat");
11806         
11807         System.out.println("Editing file: " + catalinaBatFile.getAbsolutePath());
11808         
11809         Boolean edited = editFile(catalinaBatFile, "^\\s*set\\s+\"JAVA_OPTS\\s*=.*-Xmx([0-9mMgG]+)", null, null, "512M", "max memory");
11810         if (edited == null) {
11811           addToFile(catalinaBatFile, "\nset \"JAVA_OPTS=-server -Xmx512M\"\n", 65, "max memory");
11812         }
11813       }
11814       
11815       {
11816         File catalinaShFile = new File(this.untarredTomcatDir.getAbsolutePath() + File.separator + "bin" + File.separator + "catalina.sh");
11817         
11818         System.out.println("Editing file: " + catalinaShFile.getAbsolutePath());
11819 
11820         Boolean edited = editFile(catalinaShFile, "^\\s*JAVA_OPTS\\s*=\".*-Xmx([0-9mMgG]+)", null, null, "512M", "max memory");
11821         if (edited == null) {
11822           addToFile(catalinaShFile, "\nJAVA_OPTS=\"-server -Xmx512M\"\n", 65, "max memory");
11823         }
11824       }
11825     }      
11826     
11827     
11828     if (!GrouperInstallerUtils.isWindows()) {
11829 
11830       System.out.print("Do you want to set tomcat scripts to executable (t|f)? [t]: ");
11831       boolean setTomcatFiles = readFromStdInBoolean(true, "grouperInstaller.autorun.setTomcatScriptsToExecutable");
11832       
11833       //GrouperInstallerUtils.toSet("catalina.sh", "startup.sh", "shutdown.sh");
11834       Set<String> shFileNames = new HashSet<String>();
11835 
11836       File binDir = new File(this.untarredTomcatDir.getAbsolutePath() + File.separator + "bin");
11837 
11838       //get all sh files, doing wildcards doesnt work
11839       for (File file : binDir.listFiles()) {
11840         String fileName = GrouperInstallerUtils.defaultString(file.getName());
11841         if (file.isFile() && fileName.endsWith(".sh")) {
11842           shFileNames.add(fileName);
11843         }
11844       }
11845 
11846       if (setTomcatFiles) {
11847       
11848         for (String command : shFileNames) {
11849           List<String> commands = new ArrayList<String>();
11850           
11851           commands.add("chmod");
11852           commands.add("+x");
11853           //have to do * since all the  sh files need chmod
11854           commands.add(this.untarredTomcatDir.getAbsolutePath() + File.separator + "bin" + File.separator + command);
11855     
11856           System.out.println("Making tomcat file executable with command: " + convertCommandsIntoCommand(commands) + "\n");
11857     
11858           CommandResult commandResult = GrouperInstallerUtils.execCommand(
11859               GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
11860               new File(this.untarredTomcatDir.getAbsolutePath() + File.separator + "bin"), null, true);
11861           
11862           if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
11863             System.out.println("stderr: " + commandResult.getErrorText());
11864           }
11865           if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
11866             System.out.println("stdout: " + commandResult.getOutputText());
11867           }
11868         }
11869       }
11870       
11871       Set<File> shFiles = new LinkedHashSet<File>();
11872       for (String shFileName : shFileNames) {
11873         shFiles.add(new File(shFileName));
11874       }
11875       
11876       dos2unix(shFiles, "tomcat sh files", "OnTomcatFiles");
11877 
11878     }
11879       
11880     //see what the current ports are
11881     this.tomcatHttpPort = -1;
11882     
11883     File serverXmlFile = new File(this.untarredTomcatDir.getAbsolutePath() + File.separator + "conf" + File.separator + "server.xml");
11884     
11885     int shutdownPort = GrouperInstallerUtils.xpathEvaluateAttributeInt(serverXmlFile, "/Server", "port", -1);
11886     
11887     int originalShutdownPort = shutdownPort;
11888     
11889     //  /Server/Service/Connector <Connector port="8080" protocol="HTTP/1.1" 
11890     this.tomcatHttpPort = GrouperInstallerUtils.xpathEvaluateAttributeInt(serverXmlFile, "/Server/Service/Connector[@protocol='HTTP/1.1']", "port", -1);
11891 
11892     int originalTomcatHttpPort = this.tomcatHttpPort;
11893 
11894     // <Connector port="8009" protocol="AJP/1.3" redirectPort="8443" />
11895     int jkPort = GrouperInstallerUtils.xpathEvaluateAttributeInt(serverXmlFile, "/Server/Service/Connector[@protocol='AJP/1.3']", "port", -1);
11896 
11897     int originalJkPort = jkPort;
11898     
11899     String portsCommaSeparated = GrouperInstallerUtils.propertiesValue("grouperInstaller.default.tomcatPorts", false);
11900     if (!GrouperInstallerUtils.isBlank(portsCommaSeparated)) {
11901       
11902       String[] portsStrings = GrouperInstallerUtils.splitTrim(portsCommaSeparated, ",");
11903       
11904       if (portsStrings.length != 3) {
11905         throw new RuntimeException("Why is grouperInstaller.default.tomcatPorts from grouper.installer.properties not 3 ints comma separated? " + portsCommaSeparated);
11906       }
11907       
11908       this.tomcatHttpPort = GrouperInstallerUtils.intValue(portsStrings[0]);
11909       jkPort = GrouperInstallerUtils.intValue(portsStrings[1]);
11910       shutdownPort = GrouperInstallerUtils.intValue(portsStrings[2]);
11911       
11912     }
11913     
11914     while(true) {
11915       System.out.print("What ports do you want tomcat to run on (HTTP, JK, shutdown): [" + this.tomcatHttpPort + ", " + jkPort + ", " + shutdownPort + "]: ");
11916       
11917       String ports = readFromStdIn("grouperInstaller.autorun.tomcatPorts");
11918       
11919       if (GrouperInstallerUtils.isBlank(ports)) {
11920         if (this.tomcatHttpPort == originalTomcatHttpPort && jkPort == originalJkPort && shutdownPort == originalShutdownPort) {
11921           break;
11922         }
11923       } else {
11924         String[] portsArray = GrouperInstallerUtils.splitTrim(ports, ",");
11925         if (GrouperInstallerUtils.length(portsArray) == 3) {
11926           for (String portString : portsArray) {
11927             try {
11928               GrouperInstallerUtils.intValue(portString);
11929             } catch (Exception e) {
11930               continue;
11931             }
11932           }
11933         } else {
11934           continue;
11935         }
11936         //ok, we have three integer entries
11937         this.tomcatHttpPort = GrouperInstallerUtils.intValue(portsArray[0]);
11938         jkPort = GrouperInstallerUtils.intValue(portsArray[1]);
11939         shutdownPort = GrouperInstallerUtils.intValue(portsArray[2]);
11940       }
11941       
11942       if (!GrouperInstallerUtils.portAvailable(this.tomcatHttpPort, this.defaultIpAddress)) {
11943         System.out.print("The tomcat HTTP port is in use or unavailable: " + this.tomcatHttpPort + ", do you want to pick different ports? (t|f): ");
11944         boolean pickDifferentPorts = readFromStdInBoolean(null, "grouperInstaller.autorun.pickDifferentPortIfInUse");
11945         if (pickDifferentPorts) {
11946           continue;
11947         }
11948       }
11949       if (!GrouperInstallerUtils.portAvailable(jkPort, this.defaultIpAddress)) {
11950         System.out.print("The tomcat JK port is in use or unavailable: " + this.tomcatHttpPort + ", do you want to pick different ports? (t|f): ");
11951         boolean pickDifferentPorts = readFromStdInBoolean(null, "grouperInstaller.autorun.pickDifferentPortIfInUse");
11952         if (pickDifferentPorts) {
11953           continue;
11954         }
11955       }
11956       
11957       System.out.println("Editing tomcat config file: " + serverXmlFile.getAbsolutePath());
11958       //lets edit the file
11959       //<Connector port="8080" protocol="HTTP/1.1" 
11960       editFile(serverXmlFile, "port=\"([\\d]+)\"", new String[]{"<Connector", "protocol=\"HTTP/1.1\""}, 
11961           new String[]{"SSLEnabled=\"true\""}, Integer.toString(this.tomcatHttpPort), "tomcat HTTP port");
11962       //<Connector port="8009" protocol="AJP/1.3" redirectPort="8443" />
11963       editFile(serverXmlFile, "port=\"([\\d]+)\"", new String[]{"<Connector", "protocol=\"AJP/1.3\""}, null, Integer.toString(jkPort), "tomcat JK port");
11964       //<Server port="8005" shutdown="SHUTDOWN">
11965       editFile(serverXmlFile, "port=\"([\\d]+)\"", new String[]{"<Server", "shutdown=\"SHUTDOWN\""}, null, Integer.toString(shutdownPort), "tomcat shutdown port");
11966       break;
11967     }
11968 
11969     configureTomcatUriEncoding(serverXmlFile);
11970     
11971   }
11972   
11973   /**
11974    * 
11975    */
11976   private void configureTomee() {
11977     
11978     System.out.print("Do you want to set the tomee memory limit (t|f)? [t]: ");
11979     boolean setTomeeMemory = readFromStdInBoolean(true, "grouperInstaller.autorun.setTomeeMemoryLimit");
11980     
11981     if (setTomeeMemory) {
11982       
11983       {
11984         File catalinaBatFile = new File(this.untarredTomeeDir.getAbsolutePath() + File.separator + "bin" + File.separator + "catalina.bat");
11985         
11986         System.out.println("Editing file: " + catalinaBatFile.getAbsolutePath());
11987         
11988         Boolean edited = editFile(catalinaBatFile, "^\\s*set\\s+\"JAVA_OPTS\\s*=.*-Xmx([0-9mMgG]+)", null, null, "512M", "max memory");
11989         if (edited == null) {
11990           addToFile(catalinaBatFile, "\nset \"JAVA_OPTS=-server -Xmx512M\"\n", 65, "max memory");
11991         }
11992       }
11993       
11994       {
11995         File catalinaShFile = new File(this.untarredTomeeDir.getAbsolutePath() + File.separator + "bin" + File.separator + "catalina.sh");
11996         
11997         System.out.println("Editing file: " + catalinaShFile.getAbsolutePath());
11998 
11999         Boolean edited = editFile(catalinaShFile, "^\\s*JAVA_OPTS\\s*=\".*-Xmx([0-9mMgG]+)", null, null, "512M", "max memory");
12000         if (edited == null) {
12001           addToFile(catalinaShFile, "\nJAVA_OPTS=\"-server -Xmx512M\"\n", 65, "max memory");
12002         }
12003       }
12004     }      
12005     
12006     
12007     if (!GrouperInstallerUtils.isWindows()) {
12008 
12009       System.out.print("Do you want to set tomee scripts to executable (t|f)? [t]: ");
12010       boolean setTomeeFiles = readFromStdInBoolean(true, "grouperInstaller.autorun.setTomeeScriptsToExecutable");
12011       
12012       //GrouperInstallerUtils.toSet("catalina.sh", "startup.sh", "shutdown.sh");
12013       Set<String> shFileNames = new HashSet<String>();
12014 
12015       File binDir = new File(this.untarredTomeeDir.getAbsolutePath() + File.separator + "bin");
12016 
12017       //get all sh files, doing wildcards doesnt work
12018       for (File file : binDir.listFiles()) {
12019         String fileName = GrouperInstallerUtils.defaultString(file.getName());
12020         if (file.isFile() && fileName.endsWith(".sh")) {
12021           shFileNames.add(fileName);
12022         }
12023       }
12024 
12025       if (setTomeeFiles) {
12026       
12027         for (String command : shFileNames) {
12028           List<String> commands = new ArrayList<String>();
12029           
12030           commands.add("chmod");
12031           commands.add("+x");
12032           //have to do * since all the  sh files need chmod
12033           commands.add(this.untarredTomeeDir.getAbsolutePath() + File.separator + "bin" + File.separator + command);
12034     
12035           System.out.println("Making tomee file executable with command: " + convertCommandsIntoCommand(commands) + "\n");
12036     
12037           CommandResult commandResult = GrouperInstallerUtils.execCommand(
12038               GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
12039               new File(this.untarredTomeeDir.getAbsolutePath() + File.separator + "bin"), null, true);
12040           
12041           if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
12042             System.out.println("stderr: " + commandResult.getErrorText());
12043           }
12044           if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
12045             System.out.println("stdout: " + commandResult.getOutputText());
12046           }
12047         }
12048       }
12049       
12050       Set<File> shFiles = new LinkedHashSet<File>();
12051       for (String shFileName : shFileNames) {
12052         shFiles.add(new File(shFileName));
12053       }
12054       
12055       dos2unix(shFiles, "tomee sh files", "OnTomeeFiles");
12056 
12057     }
12058       
12059     //see what the current ports are
12060     this.tomeeHttpPort = -1;
12061     
12062     File serverXmlFile = new File(this.untarredTomeeDir.getAbsolutePath() + File.separator + "conf" + File.separator + "server.xml");
12063     
12064     int shutdownPort = GrouperInstallerUtils.xpathEvaluateAttributeInt(serverXmlFile, "/Server", "port", -1);
12065     
12066     int originalShutdownPort = shutdownPort;
12067     
12068     //  /Server/Service/Connector <Connector port="8080" protocol="HTTP/1.1" 
12069     this.tomeeHttpPort = GrouperInstallerUtils.xpathEvaluateAttributeInt(serverXmlFile, "/Server/Service/Connector[@protocol='HTTP/1.1']", "port", -1);
12070 
12071     int originalTomeeHttpPort = this.tomeeHttpPort;
12072 
12073     // <Connector port="8009" protocol="AJP/1.3" redirectPort="8443" />
12074     int jkPort = GrouperInstallerUtils.xpathEvaluateAttributeInt(serverXmlFile, "/Server/Service/Connector[@protocol='AJP/1.3']", "port", -1);
12075 
12076     int originalJkPort = jkPort;
12077     
12078     String portsCommaSeparated = GrouperInstallerUtils.propertiesValue("grouperInstaller.default.tomeePorts", false);
12079     if (!GrouperInstallerUtils.isBlank(portsCommaSeparated)) {
12080       
12081       String[] portsStrings = GrouperInstallerUtils.splitTrim(portsCommaSeparated, ",");
12082       
12083       if (portsStrings.length != 3) {
12084         throw new RuntimeException("Why is grouperInstaller.default.tomeePorts from grouper.installer.properties not 3 ints comma separated? " + portsCommaSeparated);
12085       }
12086       
12087       this.tomeeHttpPort = GrouperInstallerUtils.intValue(portsStrings[0]);
12088       jkPort = GrouperInstallerUtils.intValue(portsStrings[1]);
12089       shutdownPort = GrouperInstallerUtils.intValue(portsStrings[2]);
12090       
12091     }
12092     
12093     while(true) {
12094       System.out.print("What ports do you want tomee to run on (HTTP, JK, shutdown): [" + this.tomeeHttpPort + ", " + jkPort + ", " + shutdownPort + "]: ");
12095       
12096       String ports = readFromStdIn("grouperInstaller.autorun.tomeePorts");
12097       
12098       if (GrouperInstallerUtils.isBlank(ports)) {
12099         if (this.tomeeHttpPort == originalTomeeHttpPort && jkPort == originalJkPort && shutdownPort == originalShutdownPort) {
12100           break;
12101         }
12102       } else {
12103         String[] portsArray = GrouperInstallerUtils.splitTrim(ports, ",");
12104         if (GrouperInstallerUtils.length(portsArray) == 3) {
12105           for (String portString : portsArray) {
12106             try {
12107               GrouperInstallerUtils.intValue(portString);
12108             } catch (Exception e) {
12109               continue;
12110             }
12111           }
12112         } else {
12113           continue;
12114         }
12115         //ok, we have three integer entries
12116         this.tomeeHttpPort = GrouperInstallerUtils.intValue(portsArray[0]);
12117         jkPort = GrouperInstallerUtils.intValue(portsArray[1]);
12118         shutdownPort = GrouperInstallerUtils.intValue(portsArray[2]);
12119       }
12120       
12121       if (!GrouperInstallerUtils.portAvailable(this.tomeeHttpPort, this.defaultIpAddress)) {
12122         System.out.print("The tomee HTTP port is in use or unavailable: " + this.tomeeHttpPort + ", do you want to pick different ports? (t|f): ");
12123         boolean pickDifferentPorts = readFromStdInBoolean(null, "grouperInstaller.autorun.pickDifferentPortIfInUse");
12124         if (pickDifferentPorts) {
12125           continue;
12126         }
12127       }
12128       if (!GrouperInstallerUtils.portAvailable(jkPort, this.defaultIpAddress)) {
12129         System.out.print("The tomee JK port is in use or unavailable: " + this.tomeeHttpPort + ", do you want to pick different ports? (t|f): ");
12130         boolean pickDifferentPorts = readFromStdInBoolean(null, "grouperInstaller.autorun.pickDifferentPortIfInUse");
12131         if (pickDifferentPorts) {
12132           continue;
12133         }
12134       }
12135       
12136       System.out.println("Editing tomee config file: " + serverXmlFile.getAbsolutePath());
12137       //lets edit the file
12138       //<Connector port="8080" protocol="HTTP/1.1" 
12139       editFile(serverXmlFile, "port=\"([\\d]+)\"", new String[]{"<Connector", "protocol=\"HTTP/1.1\""}, 
12140           new String[]{"SSLEnabled=\"true\""}, Integer.toString(this.tomcatHttpPort), "tomee HTTP port");
12141       //<Connector port="8009" protocol="AJP/1.3" redirectPort="8443" />
12142       editFile(serverXmlFile, "port=\"([\\d]+)\"", new String[]{"<Connector", "protocol=\"AJP/1.3\""}, null, Integer.toString(jkPort), "tomee JK port");
12143       //<Server port="8005" shutdown="SHUTDOWN">
12144       editFile(serverXmlFile, "port=\"([\\d]+)\"", new String[]{"<Server", "shutdown=\"SHUTDOWN\""}, null, Integer.toString(shutdownPort), "tomee shutdown port");
12145       break;
12146     }
12147 
12148     configureTomcatUriEncoding(serverXmlFile);
12149     
12150   }
12151   /**
12152    * @param serverXmlFile
12153    */
12154   public void configureTomcatUriEncoding(File serverXmlFile) {
12155     //set encoding for connectors
12156     //  /Server/Service/Connector <Connector port="8080" protocol="HTTP/1.1" 
12157     String uriEncodingHttp = GrouperInstallerUtils.xpathEvaluateAttribute(serverXmlFile, 
12158         "/Server/Service/Connector[@protocol='HTTP/1.1']", "URIEncoding");
12159     
12160     // <Connector port="8009" protocol="AJP/1.3" redirectPort="8443" />
12161     String uriEncodingAjp = GrouperInstallerUtils.xpathEvaluateAttribute(serverXmlFile, 
12162         "/Server/Service/Connector[@protocol='AJP/1.3']", "URIEncoding");
12163 
12164     if (!GrouperInstallerUtils.equals(uriEncodingAjp, "UTF-8") || !GrouperInstallerUtils.equals(uriEncodingHttp, "UTF-8")) {
12165 
12166       boolean defaultSetUriEncoding = GrouperInstallerUtils.propertiesValueBoolean("grouperInstaller.default.ui.setTomcatUriEncoding", true, false);
12167       System.out.print("Do you want to set URIEncoding to UTF-8 in tomcat server.xml <Connector> elements (t|f)? [" 
12168           + (defaultSetUriEncoding ? "t" : "f") + "]: ");
12169       boolean assignUriEncoding = readFromStdInBoolean(defaultSetUriEncoding, "grouperInstaller.autorun.setUriEncodingToUtf8inServerXml");
12170 
12171       if (assignUriEncoding) {
12172         
12173         if (!GrouperInstallerUtils.equals(uriEncodingAjp, "UTF-8")) {
12174           editFile(serverXmlFile, "URIEncoding=\"([^\"]+)\"", new String[]{"<Connector", "protocol=\"AJP/1.3\""}, 
12175               new String[]{"SSLEnabled=\"true\""}, "UTF-8", "tomcat URIEncoding attribute for element <Connector AJP", true, "URIEncoding");
12176           
12177         }
12178         
12179         if (!GrouperInstallerUtils.equals(uriEncodingHttp, "UTF-8")) {
12180           editFile(serverXmlFile, "URIEncoding=\"([^\"]+)\"", new String[]{"<Connector", "protocol=\"HTTP/1.1\""}, 
12181               new String[]{"SSLEnabled=\"true\""}, "UTF-8", "tomcat URIEncoding attribute for element <Connector HTTP", true, "URIEncoding");
12182           
12183         }
12184       }
12185 
12186     }
12187   }
12188 
12189   /**
12190    * 
12191    * @param newEhcacheExampleFile
12192    * @param existingEhcacheExampleFile
12193    * @param existingEhcacheFile
12194    */
12195   public static void mergeEhcacheXmlFiles(File newEhcacheExampleFile, File existingEhcacheExampleFile, File existingEhcacheFile) {
12196     
12197     try {
12198       //lets get the differences of the existing ehcache file and the existing ehcache example file
12199       DocumentBuilderFactory domFactory = GrouperInstallerUtils.xmlDocumentBuilderFactory();
12200       DocumentBuilder builder = domFactory.newDocumentBuilder();
12201       Document existingEhcacheDoc = builder.parse(existingEhcacheFile);
12202       Document existingEhcacheExampleDoc = builder.parse(existingEhcacheExampleFile);
12203 
12204       Element existingDocumentElement = existingEhcacheDoc.getDocumentElement();
12205       Element existingExampleDocumentElement = existingEhcacheExampleDoc.getDocumentElement();
12206 
12207       Map<String, String> diskStoreDifferences = null;
12208       
12209       {
12210         Element existingDiskStoreElement = (Element)existingDocumentElement.getElementsByTagName("diskStore").item(0);
12211         Element existingExampleDiskStoreElement = (Element)existingExampleDocumentElement.getElementsByTagName("diskStore").item(0);
12212         diskStoreDifferences = xmlNodeAttributeDifferences(existingExampleDiskStoreElement, existingDiskStoreElement);
12213       }
12214 
12215       Map<String, String> defaultCacheDifferences = null;
12216 
12217       {
12218         Element existingDefaultCacheElement = (Element)existingDocumentElement.getElementsByTagName("defaultCache").item(0);
12219         Element existingExampleDefaultCacheElement = (Element)existingExampleDocumentElement.getElementsByTagName("defaultCache").item(0);
12220         defaultCacheDifferences = xmlNodeAttributeDifferences(existingExampleDefaultCacheElement, existingDefaultCacheElement);
12221         
12222       }
12223 
12224       XPath xpath = XPathFactory.newInstance().newXPath();
12225       
12226       //map of cache name to the differences in the attributes of the cache
12227       Map<String, Map<String, String>> cacheDifferencesByCacheName = new LinkedHashMap<String, Map<String, String>>();
12228       
12229       {
12230         NodeList existingCacheNodeList = existingDocumentElement.getElementsByTagName("cache");
12231         
12232         //loop through all the caches
12233         for (int i=0;i<existingCacheNodeList.getLength(); i++) {
12234           
12235           Element existingCacheElement = (Element)existingCacheNodeList.item(i);
12236 
12237           String cacheName = existingCacheElement.getAttribute("name");
12238 
12239           //find the example cache
12240           XPathExpression expr = xpath.compile("cache[@name='" + cacheName + "']");
12241           Element existingExampleCacheElement = (Element)expr.evaluate(existingExampleDocumentElement, XPathConstants.NODE);
12242 
12243           //see if they differ
12244           Map<String, String> differences = xmlNodeAttributeDifferences(existingExampleCacheElement, existingCacheElement);
12245           
12246           if (differences != null) {
12247             cacheDifferencesByCacheName.put(cacheName, differences);
12248           }
12249         }
12250         
12251         //note, dont worry if there were caches in the example that werent in the configured one
12252         
12253       }      
12254       
12255       //lets see if there are any other nodes
12256       Set<Element> otherNodes = new LinkedHashSet<Element>();
12257       {
12258         NodeList nodeList = existingDocumentElement.getChildNodes();
12259         
12260         for (int i=0;i<nodeList.getLength();i++) {
12261           Node node = nodeList.item(i);
12262           if (node instanceof Element) {
12263             Element nodeElement = (Element)node;
12264             String nodeName = nodeElement.getNodeName();
12265             if (!GrouperInstallerUtils.equals(nodeName, "cache")
12266                 && !GrouperInstallerUtils.equals(nodeName, "defaultCache")
12267                 && !GrouperInstallerUtils.equals(nodeName, "diskStore")) {
12268               otherNodes.add(nodeElement);
12269             }
12270           }
12271         }
12272       }
12273       
12274       //lets copy the new example to both the example and the configured ehcache file
12275       //assume this is already backed up
12276       GrouperInstallerUtils.copyFile(newEhcacheExampleFile, existingEhcacheExampleFile, true);
12277       GrouperInstallerUtils.copyFile(newEhcacheExampleFile, existingEhcacheFile, true);
12278 
12279       //now lets do our edits
12280       if (GrouperInstallerUtils.length(diskStoreDifferences) > 0) {
12281         
12282         for (String attributeName : diskStoreDifferences.keySet()) {
12283 
12284           String attributeValue = diskStoreDifferences.get(attributeName);
12285 
12286           editXmlFileAttribute(existingEhcacheFile, "diskStore", null, attributeName, attributeValue, 
12287               "ehcache diskStore attribute '" + attributeName + "'");
12288           
12289         }
12290       }
12291       
12292       if (GrouperInstallerUtils.length(defaultCacheDifferences) > 0) {
12293         
12294         for (String attributeName : defaultCacheDifferences.keySet()) {
12295           
12296           String attributeValue = defaultCacheDifferences.get(attributeName);
12297 
12298           editXmlFileAttribute(existingEhcacheFile, "defaultCache", null, attributeName, attributeValue, 
12299               "ehcache defaultCache attribute '" + attributeName + "'");
12300 
12301         }
12302       }
12303 
12304       if (GrouperInstallerUtils.length(cacheDifferencesByCacheName) > 0) {
12305 
12306         existingEhcacheDoc = builder.parse(existingEhcacheFile);
12307         existingDocumentElement = existingEhcacheDoc.getDocumentElement();
12308         
12309         for (String cacheName : cacheDifferencesByCacheName.keySet()) {
12310 
12311           //see if the name exists
12312           //find the example cache
12313           XPathExpression expr = xpath.compile("cache[@name='" + cacheName + "']");
12314 
12315           Element existingCacheElement = (Element)expr.evaluate(existingDocumentElement, XPathConstants.NODE);
12316 
12317           Map<String, String> attributeMap = cacheDifferencesByCacheName.get(cacheName);
12318 
12319           //it exists
12320           if (existingCacheElement != null) {
12321 
12322             Map<String, String> expectedAttribute = new HashMap<String, String>();
12323 
12324             expectedAttribute.put("name", cacheName);
12325             
12326             for (String attributeName : attributeMap.keySet()) {
12327 
12328               String attributeValue = attributeMap.get(attributeName);
12329 
12330               editXmlFileAttribute(existingEhcacheFile, "cache", expectedAttribute, attributeName, attributeValue, 
12331                   "ehcache cache name=" + cacheName + " attribute '" + attributeName + "'");
12332             }
12333           } else {
12334 
12335               String fileContents = GrouperInstallerUtils.readFileIntoString(existingEhcacheFile);
12336 
12337               String newline = GrouperInstallerUtils.newlineFromFile(fileContents);
12338 
12339               int lastTagStart = fileContents.lastIndexOf("</ehcache>");
12340               
12341               if (lastTagStart == -1) {
12342                 throw new RuntimeException("Why is </ehcache> not found???? " + fileContents);
12343               }
12344 
12345               String tag = GrouperInstallerUtils.xmlElementToXml("cache", null, attributeMap);
12346 //              sdf
12347               String newFileContents = fileContents.substring(0, lastTagStart) + tag + newline 
12348                   + fileContents.substring(lastTagStart, fileContents.length());
12349 
12350               System.out.println(" - adding ehcache cache " + cacheName);
12351 
12352               GrouperInstallerUtils.saveStringIntoFile(existingEhcacheFile, newFileContents);
12353 
12354           }
12355 
12356         }
12357       }
12358 
12359       if (GrouperInstallerUtils.length(otherNodes) > 0) {
12360         String fileContents = GrouperInstallerUtils.readFileIntoString(existingEhcacheFile);
12361         
12362         String newline = GrouperInstallerUtils.newlineFromFile(fileContents);
12363 
12364         StringBuilder otherNodesStringBuilder = new StringBuilder();
12365         for (Element element : otherNodes) {
12366           String elementString = GrouperInstallerUtils.xmlToString(element);
12367           // take out the xml header: <?xml version="1.0" encoding="UTF-8"?>
12368           
12369           int elementStart = elementString.indexOf("<" + element.getNodeName());
12370           
12371           elementString = elementString.substring(elementStart);
12372           
12373           otherNodesStringBuilder.append(elementString).append(newline);
12374           System.out.println(" - adding element " + element.getTagName());
12375         }
12376 
12377         int lastTagStart = fileContents.lastIndexOf("</ehcache>");
12378         
12379         if (lastTagStart == -1) {
12380           throw new RuntimeException("Why is </ehcache> not found???? " + fileContents);
12381         }
12382 
12383         String newFileContents = fileContents.substring(0, lastTagStart) + otherNodesStringBuilder.toString()
12384             + fileContents.substring(lastTagStart, fileContents.length());
12385 
12386         GrouperInstallerUtils.saveStringIntoFile(existingEhcacheFile, newFileContents);
12387 
12388       }
12389 
12390 
12391       // test the new file, look for things
12392       existingEhcacheDoc = builder.parse(existingEhcacheFile);
12393       existingDocumentElement = existingEhcacheDoc.getDocumentElement();
12394 
12395       if (GrouperInstallerUtils.length(diskStoreDifferences) > 0) {
12396         Element existingDiskStoreElement = (Element)existingDocumentElement.getElementsByTagName("diskStore").item(0);
12397         for (String attributeName : diskStoreDifferences.keySet()) {
12398           String attributeValue = diskStoreDifferences.get(attributeName);
12399           if (!GrouperInstallerUtils.equals(attributeValue, existingDiskStoreElement.getAttribute(attributeName))) {
12400             throw new RuntimeException("Why is diskStore attribute " + attributeName + " not '" + attributeValue + "'" 
12401                 + existingEhcacheFile.getAbsolutePath());
12402           }
12403         }
12404       }
12405       
12406       if (GrouperInstallerUtils.length(defaultCacheDifferences) > 0) {
12407         Element existingDefaultCacheElement = (Element)existingDocumentElement.getElementsByTagName("defaultCache").item(0);
12408         for (String attributeName : defaultCacheDifferences.keySet()) {
12409           String attributeValue = defaultCacheDifferences.get(attributeName);
12410           if (!GrouperInstallerUtils.equals(attributeValue, existingDefaultCacheElement.getAttribute(attributeName))) {
12411             throw new RuntimeException("Why is defaultCache attribute " + attributeName + " not '" + attributeValue + "'" 
12412                 + existingEhcacheFile.getAbsolutePath());
12413           }
12414         }
12415       }
12416 
12417       if (GrouperInstallerUtils.length(cacheDifferencesByCacheName) > 0) {
12418         for (String cacheName : cacheDifferencesByCacheName.keySet()) {
12419 
12420           //see if the name exists
12421           //find the example cache
12422           XPathExpression expr = xpath.compile("cache[@name='" + cacheName + "']");
12423           Element existingCacheElement = (Element)expr.evaluate(existingDocumentElement, XPathConstants.NODE);
12424 
12425           Map<String, String> attributeMap = cacheDifferencesByCacheName.get(cacheName);
12426           
12427           for (String attributeName : attributeMap.keySet()) {
12428             
12429             String attributeValue = attributeMap.get(attributeName);
12430 
12431             if (!GrouperInstallerUtils.equals(attributeValue, existingCacheElement.getAttribute(attributeName))) {
12432               throw new RuntimeException("Why is cache " + cacheName + " attribute " + attributeName + " not '" + attributeValue + "'" 
12433                   + existingEhcacheFile.getAbsolutePath());
12434             }
12435             
12436           }
12437         }
12438       }
12439 
12440       if (GrouperInstallerUtils.length(otherNodes) > 0) {
12441         for (Element element : otherNodes) {
12442           
12443           NodeList nodeList = existingDocumentElement.getElementsByTagName(element.getNodeName());
12444           if (nodeList == null || nodeList.getLength() == 0 ) {
12445             throw new RuntimeException("Why is new element not there? " + element.getTagName() + ", "
12446                 + existingEhcacheFile.getAbsolutePath());
12447           }
12448         }
12449       }
12450 
12451     } catch (Exception e) {
12452       throw new RuntimeException(e.getMessage(), e);
12453     }
12454   }
12455 
12456   /**
12457    * 
12458    * @param newEhcacheExampleFile
12459    * @param existingEhcacheExampleFile
12460    * @param existingEhcacheFile
12461    * @return hasMerging
12462    */
12463   @SuppressWarnings("unused")
12464   private static boolean mergeEhcacheXmlFiles_XML_NOT_USED(File newEhcacheExampleFile, File existingEhcacheExampleFile, 
12465       File existingEhcacheFile) {
12466     
12467     boolean hasMerging = false;
12468     
12469     try {
12470       //lets get the differences of the existing ehcache file and the existing ehcache example file
12471       DocumentBuilderFactory domFactory = GrouperInstallerUtils.xmlDocumentBuilderFactory();
12472       DocumentBuilder builder = domFactory.newDocumentBuilder();
12473       Document existingEhcacheDoc = builder.parse(existingEhcacheFile);
12474       Document existingEhcacheExampleDoc = builder.parse(existingEhcacheExampleFile);
12475 
12476       Element existingDocumentElement = existingEhcacheDoc.getDocumentElement();
12477       Element existingExampleDocumentElement = existingEhcacheExampleDoc.getDocumentElement();
12478 
12479       Map<String, String> diskStoreDifferences = null;
12480       
12481       {
12482         Element existingDiskStoreElement = (Element)existingDocumentElement.getElementsByTagName("diskStore").item(0);
12483         Element existingExampleDiskStoreElement = (Element)existingExampleDocumentElement.getElementsByTagName("diskStore").item(0);
12484         diskStoreDifferences = xmlNodeAttributeDifferences(existingExampleDiskStoreElement, existingDiskStoreElement);
12485       }
12486 
12487       Map<String, String> defaultCacheDifferences = null;
12488 
12489       {
12490         Element existingDefaultCacheElement = (Element)existingDocumentElement.getElementsByTagName("defaultCache").item(0);
12491         Element existingExampleDefaultCacheElement = (Element)existingExampleDocumentElement.getElementsByTagName("defaultCache").item(0);
12492         defaultCacheDifferences = xmlNodeAttributeDifferences(existingExampleDefaultCacheElement, existingDefaultCacheElement);
12493         
12494       }
12495 
12496       XPath xpath = XPathFactory.newInstance().newXPath();
12497       
12498       //map of cache name to the differences in the attributes of the cache
12499       Map<String, Map<String, String>> cacheDifferencesByCacheName = new LinkedHashMap<String, Map<String, String>>();
12500       
12501       {
12502         NodeList existingCacheNodeList = existingDocumentElement.getElementsByTagName("cache");
12503         
12504         //loop through all the caches
12505         for (int i=0;i<existingCacheNodeList.getLength(); i++) {
12506           
12507           Element existingCacheElement = (Element)existingCacheNodeList.item(i);
12508 
12509           String cacheName = existingCacheElement.getAttribute("name");
12510 
12511           //find the example cache
12512           XPathExpression expr = xpath.compile("cache[@name='" + cacheName + "']");
12513           Element existingExampleCacheElement = (Element)expr.evaluate(existingExampleDocumentElement, XPathConstants.NODE);
12514 
12515           //see if they differ
12516           Map<String, String> differences = xmlNodeAttributeDifferences(existingExampleCacheElement, existingCacheElement);
12517           
12518           if (differences != null) {
12519             cacheDifferencesByCacheName.put(cacheName, differences);
12520           }
12521         }
12522         
12523         //note, dont worry if there were caches in the example that werent in the configured one
12524         
12525       }      
12526       
12527       //lets see if there are any other nodes
12528       Set<Element> otherNodes = new LinkedHashSet<Element>();
12529       {
12530         NodeList nodeList = existingDocumentElement.getChildNodes();
12531         
12532         for (int i=0;i<nodeList.getLength();i++) {
12533           Node node = nodeList.item(i);
12534           if (node instanceof Element) {
12535             Element nodeElement = (Element)node;
12536             String nodeName = nodeElement.getNodeName();
12537             if (!GrouperInstallerUtils.equals(nodeName, "cache")
12538                 && !GrouperInstallerUtils.equals(nodeName, "defaultCache")
12539                 && !GrouperInstallerUtils.equals(nodeName, "diskStore")) {
12540               otherNodes.add(nodeElement);
12541             }
12542           }
12543         }
12544       }
12545       
12546       //lets copy the new example to both the example and the configured ehcache file
12547       //assume this is already backed up
12548       GrouperInstallerUtils.copyFile(newEhcacheExampleFile, existingEhcacheExampleFile, true);
12549 
12550       //this is the new existing ehcache file
12551       existingEhcacheExampleDoc = builder.parse(existingEhcacheExampleFile);
12552       existingExampleDocumentElement = existingEhcacheExampleDoc.getDocumentElement();
12553 
12554       //now lets do our edits
12555       if (GrouperInstallerUtils.length(diskStoreDifferences) > 0) {
12556         
12557         hasMerging = true;
12558 
12559         Element existingExampleDiskStoreElement = (Element)existingExampleDocumentElement.getElementsByTagName("diskStore").item(0);
12560 
12561         for (String attributeName : diskStoreDifferences.keySet()) {
12562 
12563           String attributeValue = diskStoreDifferences.get(attributeName);
12564 
12565           existingExampleDiskStoreElement.setAttribute(attributeName, attributeValue);
12566         }
12567       }
12568       
12569       if (GrouperInstallerUtils.length(defaultCacheDifferences) > 0) {
12570 
12571         hasMerging = true;
12572 
12573         Element existingExampleDefaultCacheElement = (Element)existingExampleDocumentElement.getElementsByTagName("defaultCache").item(0);
12574         
12575         for (String attributeName : defaultCacheDifferences.keySet()) {
12576           
12577           String attributeValue = defaultCacheDifferences.get(attributeName);
12578 
12579           existingExampleDefaultCacheElement.setAttribute(attributeName, attributeValue);
12580           
12581         }
12582       }
12583 
12584       if (GrouperInstallerUtils.length(cacheDifferencesByCacheName) > 0) {
12585         hasMerging = true;
12586         for (String cacheName : cacheDifferencesByCacheName.keySet()) {
12587 
12588           //see if the name exists
12589           //find the example cache
12590           XPathExpression expr = xpath.compile("cache[@name='" + cacheName + "']");
12591           Element existingExampleCacheElement = (Element)expr.evaluate(existingExampleDocumentElement, XPathConstants.NODE);
12592 
12593           Map<String, String> attributeMap = cacheDifferencesByCacheName.get(cacheName);
12594           
12595           //it exists
12596           if (existingExampleCacheElement != null) {
12597             
12598             for (String attributeName : attributeMap.keySet()) {
12599               
12600               String attributeValue = attributeMap.get(attributeName);
12601               existingExampleCacheElement.setAttribute(attributeName, attributeValue);
12602               
12603             }
12604           } else {
12605             
12606             Element existingCacheElement = (Element)expr.evaluate(existingDocumentElement, XPathConstants.NODE);
12607             //move a cache from one document to another
12608             existingExampleDocumentElement.appendChild(existingCacheElement.cloneNode(true));
12609             
12610           }
12611           
12612         }
12613       }
12614 
12615       if (GrouperInstallerUtils.length(otherNodes) > 0) {
12616         hasMerging = true;
12617         for (Element element : otherNodes) {
12618           
12619           //move a cache from one document to another
12620           existingExampleDocumentElement.appendChild(element.cloneNode(true));
12621         }
12622       }
12623 
12624 //      System.out.println("Compare you old ehcache.xml with your new ehcache.xml file: " 
12625 //          + "\n  Old file: "
12626 //          + backedUpEhcacheFile.getAbsolutePath()
12627 //          + "\n  New file: " + existingEhcacheFile.getAbsolutePath());
12628 
12629       // save to file
12630       String xml = GrouperInstallerUtils.xmlToString(existingEhcacheExampleDoc);
12631       GrouperInstallerUtils.saveStringIntoFile(existingEhcacheFile, xml);
12632       
12633       // test the new file, look for things
12634       existingEhcacheDoc = builder.parse(existingEhcacheFile);
12635       existingDocumentElement = existingEhcacheDoc.getDocumentElement();
12636 
12637       if (GrouperInstallerUtils.length(diskStoreDifferences) > 0) {
12638         Element existingDiskStoreElement = (Element)existingDocumentElement.getElementsByTagName("diskStore").item(0);
12639         for (String attributeName : diskStoreDifferences.keySet()) {
12640           String attributeValue = diskStoreDifferences.get(attributeName);
12641           if (!GrouperInstallerUtils.equals(attributeValue, existingDiskStoreElement.getAttribute(attributeName))) {
12642             throw new RuntimeException("Why is diskStore attribute " + attributeName + " not '" + attributeValue + "'" 
12643                 + existingEhcacheFile.getAbsolutePath());
12644           }
12645         }
12646       }
12647       
12648       if (GrouperInstallerUtils.length(defaultCacheDifferences) > 0) {
12649         Element existingDefaultCacheElement = (Element)existingDocumentElement.getElementsByTagName("defaultCache").item(0);
12650         for (String attributeName : defaultCacheDifferences.keySet()) {
12651           String attributeValue = defaultCacheDifferences.get(attributeName);
12652           if (!GrouperInstallerUtils.equals(attributeValue, existingDefaultCacheElement.getAttribute(attributeName))) {
12653             throw new RuntimeException("Why is defaultCache attribute " + attributeName + " not '" + attributeValue + "'" 
12654                 + existingEhcacheFile.getAbsolutePath());
12655           }
12656         }
12657       }
12658 
12659       if (GrouperInstallerUtils.length(cacheDifferencesByCacheName) > 0) {
12660         for (String cacheName : cacheDifferencesByCacheName.keySet()) {
12661 
12662           //see if the name exists
12663           //find the example cache
12664           XPathExpression expr = xpath.compile("cache[@name='" + cacheName + "']");
12665           Element existingCacheElement = (Element)expr.evaluate(existingDocumentElement, XPathConstants.NODE);
12666 
12667           Map<String, String> attributeMap = cacheDifferencesByCacheName.get(cacheName);
12668           
12669           for (String attributeName : attributeMap.keySet()) {
12670             
12671             String attributeValue = attributeMap.get(attributeName);
12672 
12673             if (!GrouperInstallerUtils.equals(attributeValue, existingCacheElement.getAttribute(attributeName))) {
12674               throw new RuntimeException("Why is cache " + cacheName + " attribute " + attributeName + " not '" + attributeValue + "'" 
12675                   + existingEhcacheFile.getAbsolutePath());
12676             }
12677             
12678           }
12679         }
12680       }
12681 
12682       if (GrouperInstallerUtils.length(otherNodes) > 0) {
12683         for (Element element : otherNodes) {
12684           
12685           NodeList nodeList = existingDocumentElement.getElementsByTagName(element.getNodeName());
12686           if (nodeList == null || nodeList.getLength() == 0 ) {
12687             throw new RuntimeException("Why is new element not there? " + element.getTagName() 
12688                 + existingEhcacheFile.getAbsolutePath());
12689           }
12690         }
12691       }
12692 
12693     } catch (Exception e) {
12694       throw new RuntimeException(e.getMessage(), e);
12695     }
12696     return hasMerging;
12697   }
12698   /**
12699    * 
12700    * @param baseElement
12701    * @param configuredElement
12702    * @return the map of differences
12703    */
12704   public static Map<String, String> xmlNodeAttributeDifferences(Element baseElement, Element configuredElement) {
12705     NamedNodeMap configuredNamedNodeMap = configuredElement.getAttributes();
12706     
12707     Map<String, String> result = null;
12708     
12709     //see which attributes are new or changed
12710     for (int i=0;i<configuredNamedNodeMap.getLength();i++) {
12711       Node configuredAttribute = configuredNamedNodeMap.item(i);
12712       Node baseAttribute = baseElement == null ? null : baseElement.getAttributeNode(configuredAttribute.getNodeName());
12713 
12714       String configuredValue = configuredAttribute.getNodeValue();
12715       String baseValue = baseAttribute == null ? null : baseAttribute.getNodeValue();
12716       
12717       if (!GrouperInstallerUtils.equals(configuredValue, baseValue)) {
12718         if (result == null) {
12719           result = new LinkedHashMap<String, String>();
12720         }
12721         result.put(configuredAttribute.getNodeName(), configuredValue);
12722       }
12723     }
12724     
12725     //see which ones are missing
12726     NamedNodeMap baseNamedNodeMap = baseElement == null ? null : baseElement.getAttributes();
12727     
12728     //see which attributes are new or changed
12729     for (int i=0;i<(baseNamedNodeMap == null ? 0 : baseNamedNodeMap.getLength());i++) {
12730       
12731       Node baseAttribute = configuredNamedNodeMap.item(0);
12732       Node configuredAttribute = configuredElement.getAttributeNode(baseAttribute.getNodeName());
12733 
12734       String baseValue = baseAttribute.getNodeValue();
12735       String configuredValue = configuredAttribute == null ? null : configuredAttribute.getNodeValue();
12736       
12737       if (configuredValue == null && !GrouperInstallerUtils.equals(configuredValue, baseValue)) {
12738         if (result == null) {
12739           result = new LinkedHashMap<String, String>();
12740         }
12741         result.put(baseAttribute.getNodeName(), configuredValue);
12742       }
12743     }
12744     
12745     return result;
12746   }
12747   
12748   /**
12749    * 
12750    * @return the file of the directory of API
12751    */
12752   private File downloadApi() {
12753     String urlToDownload = GrouperInstallerUtils.propertiesValue("download.server.url", true);
12754     
12755     if (!urlToDownload.endsWith("/")) {
12756       urlToDownload += "/";
12757     }
12758     urlToDownload += "release/";
12759     String apiFileName = "grouper.apiBinary-" + this.version + ".tar.gz";
12760     urlToDownload += this.version + "/" + apiFileName;
12761 
12762     File apiFile = new File(this.grouperTarballDirectoryString + apiFileName);
12763     
12764     downloadFile(urlToDownload, apiFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalApiDownloadTarEtc");
12765 
12766     return apiFile;
12767   }
12768 
12769   /**
12770    * 
12771    * @return the file of the directory of UI
12772    */
12773   private File downloadUi() {
12774     String urlToDownload = GrouperInstallerUtils.propertiesValue("download.server.url", true);
12775     
12776     if (!urlToDownload.endsWith("/")) {
12777       urlToDownload += "/";
12778     }
12779     urlToDownload += "release/";
12780 
12781     String uiFileName = "grouper.ui-" + this.version + ".tar.gz";
12782     urlToDownload += this.version + "/" + uiFileName;
12783 
12784     File uiFile = new File(this.grouperTarballDirectoryString + uiFileName);
12785     
12786     downloadFile(urlToDownload, uiFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalUiDownloadTarEtc");
12787 
12788     return uiFile;
12789   }
12790   
12791   /**
12792    * 
12793    * @return the file of the directory of WS
12794    */
12795   private File downloadWs() {
12796     
12797     String urlToDownload = GrouperInstallerUtils.propertiesValue("download.server.url", true);
12798     
12799     //String urlToDownload = "http://localhost:8085/grouper/grouperExternal/public/assets/dojo/dijit/themes/claro/images";
12800     
12801     if (!urlToDownload.endsWith("/")) {
12802       urlToDownload += "/";
12803     }
12804     urlToDownload += "release/";
12805 
12806     String wsFileName = "grouper.ws-" + this.version + ".tar.gz";
12807     urlToDownload += this.version + "/" + wsFileName;
12808 
12809     File wsFile = new File(this.grouperTarballDirectoryString + wsFileName);
12810     
12811     System.out.println("wsFile path is "+wsFile.getAbsolutePath());
12812     downloadFile(urlToDownload, wsFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalWsDownloadTarEtc");
12813 
12814     return wsFile;
12815   }
12816 
12817   /**
12818    * 
12819    * @return the file of the directory of ant
12820    */
12821   private File downloadAnt() {
12822     String urlToDownload = GrouperInstallerUtils.propertiesValue("download.server.url", true);
12823     
12824     if (!urlToDownload.endsWith("/")) {
12825       urlToDownload += "/";
12826     }
12827 
12828     urlToDownload += "downloads/tools/apache-ant-1.8.2-bin.tar.gz";
12829     
12830     File antFile = new File(this.grouperTarballDirectoryString + "apache-ant-1.8.2-bin.tar.gz");
12831     
12832     downloadFile(urlToDownload, antFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc");
12833 
12834     return antFile;
12835   }
12836 
12837   /**
12838    * 
12839    * @return the file of the directory of maven
12840    */
12841   private File downloadMaven() {
12842     String urlToDownload = GrouperInstallerUtils.propertiesValue("download.server.url", true);
12843     
12844     if (!urlToDownload.endsWith("/")) {
12845       urlToDownload += "/";
12846     }
12847 
12848     urlToDownload += "downloads/tools/apache-maven-3.6.3-bin.tar.gz";
12849     
12850     File mavenFile = new File(this.grouperTarballDirectoryString + "apache-maven-3.6.3-bin.tar.gz");
12851     
12852     downloadFile(urlToDownload, mavenFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc");
12853 
12854     return mavenFile;
12855   }
12856 
12857   /**
12858    * tomcat version
12859    */
12860   private String tomcatVersion = "8.5.87";
12861   
12862   /**
12863    * 
12864    * @return tomcat version
12865    */
12866   private String tomcatVersion() {
12867     
12868     // this is now hardcoded
12869     if (this.tomcatVersion == null) {
12870       
12871       String defaultTomcatVersion = GrouperInstallerUtils.propertiesValue("grouperInstaller.default.tomcat.version", false);
12872       defaultTomcatVersion = GrouperInstallerUtils.defaultIfBlank(defaultTomcatVersion, "8.5.87");
12873       
12874       System.out.print("Enter the tomcat version (8.5.84 or 8.5.12 or 6.0.35) [" + defaultTomcatVersion + "]: ");
12875       this.tomcatVersion = readFromStdIn("grouperInstaller.autorun.tomcat.version");
12876       
12877       this.tomcatVersion = GrouperInstallerUtils.defaultIfBlank(this.tomcatVersion, defaultTomcatVersion);
12878       
12879       if (!GrouperInstallerUtils.equals(this.tomcatVersion, "8.5.84") && !GrouperInstallerUtils.equals(this.tomcatVersion, "6.0.35")) {
12880         System.out.print("Warning: this *should* be 8.5.84 or 8.5.12 or 6.0.35, hit <Enter> to continue: ");
12881         readFromStdIn("grouperInstaller.autorun.tomcat.version.mismatch");
12882       }
12883       
12884     }
12885     
12886     return this.tomcatVersion;
12887 
12888   }
12889   
12890   /**
12891    * Copy a jar file to another file.  this perserves the file date
12892    * 
12893    * @param sourceFile
12894    * @param destinationFile
12895    * @param onlyIfDifferentContents true if only saving due to different contents.  Note, this is only for non-binary files!
12896    * @param ignoreWhitespace true to ignore whitespace in comparisons
12897    * @return true if contents were saved (thus different if param set)
12898    */
12899   public static boolean copyJarFileIfNotExists(File sourceFile, File destinationFile, boolean onlyIfDifferentContents, boolean ignoreWhitespace) {
12900     
12901     if (!sourceFile.isFile() || !sourceFile.exists()) {
12902       throw new RuntimeException("Why does this not exist???? " + sourceFile.getAbsolutePath());
12903     }
12904     
12905     if (destinationFile.isFile() && destinationFile.exists() && 
12906         GrouperInstallerUtils.equals(GrouperInstallerUtils.fileSha1(destinationFile), GrouperInstallerUtils.fileSha1(sourceFile))) {
12907       System.out.println("Skipping file that exists in destination: " + destinationFile.getAbsolutePath());
12908       return false;
12909     }
12910 
12911     if (onlyIfDifferentContents) {
12912       String sourceContents = GrouperInstallerUtils.readFileIntoString(sourceFile);
12913       return GrouperInstallerUtils.saveStringIntoFile(destinationFile, sourceContents, 
12914           onlyIfDifferentContents, ignoreWhitespace);
12915     }
12916     
12917     File destinationFolder = destinationFile.getParentFile();
12918     
12919     Set<String> relatedBaseNames = GrouperInstallerUtils.jarFileBaseNames(destinationFile.getName());
12920 
12921     boolean hasConflict = false;
12922     for (File destinationCandidateFile : destinationFolder.listFiles()) {
12923       if (!destinationCandidateFile.getName().endsWith(".jar")) {
12924         continue;
12925       }
12926       Set<String> relatedCandidateBaseNames = GrouperInstallerUtils.jarFileBaseNames(destinationCandidateFile.getName());
12927       if (GrouperInstallerUtils.containsAny(relatedBaseNames, relatedCandidateBaseNames)) {
12928         
12929         hasConflict = true;
12930       }
12931     }
12932     
12933     if (hasConflict) {
12934       List<File> relatedFiles = GrouperInstallerUtils.jarFindJar(destinationFolder, sourceFile.getName());
12935       
12936       if (GrouperInstallerUtils.length(relatedFiles) == 1) {
12937         File relatedFile = relatedFiles.iterator().next();
12938         File newerVersion = GrouperInstallerUtils.jarNewerVersion(relatedFile, sourceFile);
12939         if (newerVersion != null) {
12940           
12941           if (newerVersion.equals(sourceFile)) {
12942             System.out.println("There is a conflicting jar: " + sourceFile.getAbsolutePath());
12943             System.out.println("Deleting older jar: " + relatedFile.getAbsolutePath());
12944             GrouperInstallerUtils.fileDelete(relatedFile);
12945             System.out.println("Copying " + sourceFile.getAbsolutePath() + " to " + destinationFile.getAbsolutePath());
12946             GrouperInstallerUtils.copyFile(sourceFile, destinationFile);
12947             return true;
12948           }
12949           System.out.println("There is a conflicting jar for source: " + sourceFile.getAbsolutePath());
12950           System.out.println("Not copying to dest due to this jar is newer: " + relatedFile.getAbsolutePath());
12951           return false;
12952         }
12953         System.out.println("There is a conflicting jar, source jar: " + sourceFile.getAbsolutePath());
12954         System.out.println("Destination jar: " + destinationFile.getAbsolutePath());
12955         System.out.print("Unable to resolve conflict, resolve manually, press <enter> to continue... ");
12956         readFromStdIn("grouperInstaller.autorun.conflictingJarContinue");
12957         return false;
12958       }
12959 
12960     }
12961     
12962     System.out.println("Copying " + sourceFile.getAbsolutePath() + " to " + destinationFile.getAbsolutePath());
12963     GrouperInstallerUtils.copyFile(sourceFile, destinationFile);
12964     return true;
12965   }
12966 
12967   /**
12968    * 
12969    * @return the file of the directory of tomcat
12970    */
12971   private File downloadTomcat() {
12972     String urlToDownload = GrouperInstallerUtils.propertiesValue("download.server.url", true);
12973     
12974     if (!urlToDownload.endsWith("/")) {
12975       urlToDownload += "/";
12976     }
12977 
12978     urlToDownload += "downloads/tools/apache-tomcat-" + this.tomcatVersion() + ".tar.gz";
12979     
12980     File tomcatFile = new File(this.grouperTarballDirectoryString + "apache-tomcat-" + this.tomcatVersion() + ".tar.gz");
12981     
12982     downloadFile(urlToDownload, tomcatFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc");
12983 
12984     return tomcatFile;
12985   }
12986   
12987   public static final String TOMEE_VERSION = "7.0.9";
12988   
12989   /**
12990    * 
12991    * @return the file of the directory of tomee
12992    */
12993   private File downloadTomee() {
12994     String urlToDownload = GrouperInstallerUtils.propertiesValue("download.server.url", true);
12995     
12996     //String urlToDownload = "http://localhost:8085/grouper/grouperExternal/public/assets/dojo/dijit/themes/claro/images";
12997     
12998     if (!urlToDownload.endsWith("/")) {
12999       urlToDownload += "/";
13000     }
13001 
13002     urlToDownload += "downloads/tools/apache-tomee-" + TOMEE_VERSION + "-webprofile.tar.gz";
13003     
13004     File tomeeFile = new File(this.grouperTarballDirectoryString + "apache-tomee-" + TOMEE_VERSION + "-webprofile.tar.gz");
13005     
13006     downloadFile(urlToDownload, tomeeFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc");
13007 
13008     return tomeeFile;
13009   }
13010   
13011   private File downloadGrouperSourceTagFromGithub() {
13012     
13013     File grouperSourceCodeFile = new File(this.grouperTarballDirectoryString + "GROUPER_RELEASE_"+this.version+".tar.gz");
13014     downloadFile("https://github.com/Internet2/grouper/archive/GROUPER_RELEASE_"+this.version+".tar.gz", grouperSourceCodeFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalToolsDownloadTarEtc");
13015     return grouperSourceCodeFile;
13016   }
13017   
13018   private void deleteJarsFromLibDirs(File webInfDir) {
13019     
13020     List<File> allJarsToBeDeleted = new ArrayList<File>();
13021     
13022     List<File> libDirs = new ArrayList<File>();
13023     libDirs.add(new File(webInfDir+File.separator+"lib"));
13024     libDirs.add(new File(webInfDir+File.separator+"libUiAndDaemon"));
13025     libDirs.add(new File(webInfDir+File.separator+"libWs"));
13026     
13027 //    for (File libDir: libDirs) {
13028 //      File[] filesFromLibToBeDeleted = libDir.listFiles(new FilenameFilter() {
13029 //        
13030 //        @Override
13031 //        public boolean accept(File dir, String name) {
13032 //          if (name.startsWith("slf4j-api") && name.endsWith("jar")) {
13033 //            return true;
13034 //          }
13035 //          
13036 //          if (name.startsWith("slf4j-log4j12") && name.endsWith("jar")) {
13037 //            return true;
13038 //          }
13039 //                    
13040 //          return false;
13041 //        }
13042 //      });
13043 //      allJarsToBeDeleted.addAll(Arrays.asList(filesFromLibToBeDeleted));
13044 //    }
13045 //    
13046 //    for (File jarToBeDeleted: allJarsToBeDeleted) {
13047 //      GrouperInstallerUtils.fileDelete(jarToBeDeleted);
13048 //    }
13049     
13050   }
13051   
13052   private void downloadGrouperJarsIntoLibDirectory(File webInfDir) {
13053     String basePath = "https://oss.sonatype.org/service/local/repositories/releases/content/edu/internet2/middleware/grouper/";
13054     
13055     {
13056       File libDir = new File(webInfDir+File.separator+"lib");
13057       
13058       List<String> urlsToDownload = new ArrayList<String>();
13059       urlsToDownload.add(basePath+"grouper/"+this.version+"/grouper-"+this.version+".jar");
13060       urlsToDownload.add(basePath+"grouperClient/"+this.version+"/grouperClient-"+this.version+".jar");
13061       
13062       for (String urlToDownload: urlsToDownload) {
13063         String fileName = urlToDownload.substring(urlToDownload.lastIndexOf(File.separator)+1, urlToDownload.length());
13064         downloadFile(urlToDownload, libDir.getAbsolutePath() + File.separator+ fileName, "grouperInstaller.autorun.buildContainerUseExistingJarIfExists");
13065       }
13066     }
13067     
13068     {
13069       File libUiAndDaemonDir = new File(webInfDir+File.separator+"libUiAndDaemon");
13070       List<String> urlsToDownload = new ArrayList<String>();
13071       urlsToDownload.add(basePath+"google-apps-provisioner/"+this.version+"/google-apps-provisioner-"+this.version+".jar");
13072       urlsToDownload.add(basePath+"grouper-ui/"+this.version+"/grouper-ui-"+this.version+".jar");
13073       urlsToDownload.add(basePath+"grouper-pspng/"+this.version+"/grouper-pspng-"+this.version+".jar");
13074       urlsToDownload.add(basePath+"grouper-box/"+this.version+"/grouper-box-"+this.version+".jar");
13075       urlsToDownload.add(basePath+"grouper-duo/"+this.version+"/grouper-duo-"+this.version+".jar");
13076       urlsToDownload.add(basePath+"grouper-azure-provisioner/"+this.version+"/grouper-azure-provisioner-"+this.version+".jar");
13077       for (String urlToDownload: urlsToDownload) {
13078         String fileName = urlToDownload.substring(urlToDownload.lastIndexOf(File.separator)+1, urlToDownload.length());
13079         downloadFile(urlToDownload, libUiAndDaemonDir.getAbsolutePath() + File.separator+ fileName, "grouperInstaller.autorun.buildContainerUseExistingJarIfExists");
13080       }
13081     }
13082     
13083     {
13084       File libWsDir = new File(webInfDir+File.separator+"libWs");
13085       String wsUrlToDownload = basePath+"grouper-ws/"+this.version+"/grouper-ws-"+this.version+".jar";
13086       String wsJarfileName = wsUrlToDownload.substring(wsUrlToDownload.lastIndexOf(File.separator)+1, wsUrlToDownload.length());
13087       downloadFile(wsUrlToDownload, libWsDir.getAbsolutePath() + File.separator+ wsJarfileName, "grouperInstaller.autorun.buildContainerUseExistingJarIfExists");
13088     }
13089     
13090   }
13091 
13092   /**
13093    * add quick start subjects
13094    */
13095   private void addQuickstartSubjects() {
13096     
13097     System.out.print("Do you want to add quickstart subjects to DB (t|f)? [t]: ");
13098     boolean addQuickstartSubjects = readFromStdInBoolean(true, "grouperInstaller.autorun.addQuickstartSubjectsToDb");
13099     
13100     if (addQuickstartSubjects) {
13101 
13102       String url = GrouperInstallerUtils.propertiesValue("download.server.url", true);
13103       
13104       if (!url.endsWith("/")) {
13105         url += "/";
13106       }
13107       url += "release/" + this.version + "/subjects.sql";
13108 
13109       String subjectsSqlFileName = this.untarredApiDir.getParent() + File.separator + "subjects.sql";
13110       File subjectsSqlFile = new File(subjectsSqlFileName);
13111       downloadFile(url, subjectsSqlFileName, "grouperInstaller.autorun.useLocalApiDownloadTarEtc");
13112 
13113       List<String> commands = new ArrayList<String>();
13114       
13115       addGshCommands(commands);
13116       commands.add("-registry");
13117       commands.add("-runsqlfile");
13118       commands.add(subjectsSqlFile.getAbsolutePath());
13119       commands.add("-noprompt");
13120       
13121       System.out.println("\n##################################");
13122       System.out.println("Adding sample subjects with command: " + convertCommandsIntoCommand(commands) + "\n");
13123       
13124       CommandResult commandResult = GrouperInstallerUtils.execCommand(
13125           GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
13126           this.untarredApiDir, null, true);
13127       
13128       if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
13129         System.out.println("stderr: " + commandResult.getErrorText());
13130       }
13131       if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
13132         System.out.println("stdout: " + commandResult.getOutputText());
13133       }
13134       
13135       System.out.println("\nEnd adding sample subjects");
13136       System.out.println("##################################\n");
13137 
13138     }
13139   }
13140 
13141   /**
13142    * add quick start subjects
13143    */
13144   private void addQuickstartData() {
13145 
13146     System.out.print("Do you want to add quickstart data to registry (t|f)? [t] ");
13147     boolean addQuickstartData = readFromStdInBoolean(true, "grouperInstaller.autorun.addQuickstartData");
13148 
13149     if (addQuickstartData) {
13150       String url = GrouperInstallerUtils.propertiesValue("download.server.url", true);
13151       
13152       if (!url.endsWith("/")) {
13153         url += "/";
13154       }
13155       url += "release/" + this.version + "/quickstart.xml";
13156       String quickstartFileName = this.untarredApiDir.getParent() + File.separator + "quickstart.xml";
13157       
13158       File quickstartFile = new File(quickstartFileName);
13159       downloadFile(url, quickstartFileName, "grouperInstaller.autorun.useLocalApiDownloadTarEtc");
13160 
13161       List<String> commands = new ArrayList<String>();
13162       
13163       addGshCommands(commands);
13164       commands.add("-xmlimportold");
13165       commands.add("GrouperSystem");
13166       commands.add(quickstartFile.getAbsolutePath());
13167       commands.add("-noprompt");
13168       
13169       System.out.println("\n##################################");
13170       System.out.println("Adding quickstart data with command: " + convertCommandsIntoCommand(commands) + "\n");
13171       
13172       CommandResult commandResult = GrouperInstallerUtils.execCommand(
13173           GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
13174           this.untarredApiDir, null, true);
13175       
13176       if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
13177         System.out.println("stderr: " + commandResult.getErrorText());
13178       }
13179       if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
13180 
13181         System.out.println("stdout: " + commandResult.getOutputText());
13182       }
13183       System.out.println("\nEnd adding quickstart data");
13184       System.out.println("##################################\n");
13185 
13186     }
13187   }
13188   
13189   /**
13190    * if commands have spaces, put quotes around...
13191    * @param commands
13192    * @return the command
13193    */
13194   private static String convertCommandsIntoCommand(List<String> commands) {
13195     StringBuilder result = new StringBuilder();
13196     for (int i=0;i<GrouperInstallerUtils.length(commands); i++) {
13197       String command = GrouperInstallerUtils.defaultString(commands.get(i));
13198       
13199       //if there is a space, put quotes around command
13200       if (command.contains(" ")) {
13201         result.append("\"").append(command).append("\"");
13202       } else {
13203         result.append(command);
13204       }
13205       if (i != GrouperInstallerUtils.length(commands)-1) {
13206         result.append(" ");
13207       }
13208     }
13209     return result.toString();
13210   }
13211   
13212   /**
13213    * 
13214    */
13215   private void initDb() {
13216     System.out.print("Do you want to init the database (delete all existing grouper tables, add new ones) (t|f)? ");
13217     boolean initdb = readFromStdInBoolean(null, "grouperInstaller.autorun.deleteAndInitDatabase");
13218     
13219     if (initdb) {
13220       List<String> commands = new ArrayList<String>();
13221       
13222       addGshCommands(commands);
13223       commands.add("-registry");
13224       commands.add("-drop");
13225       commands.add("-runscript");
13226       commands.add("-noprompt");
13227       
13228       System.out.println("\n##################################");
13229       System.out.println("Initting DB with command: " + convertCommandsIntoCommand(commands) + "\n");
13230       
13231       CommandResult commandResult = GrouperInstallerUtils.execCommand(
13232           GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
13233           this.untarredApiDir, null, true);
13234       
13235       if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
13236         System.out.println("stderr: " + commandResult.getErrorText());
13237       }
13238       if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
13239 
13240         System.out.println("stdout: " + commandResult.getOutputText());
13241       }
13242       System.out.println("\nEnd Initting DB");
13243       System.out.println("##################################\n");
13244       
13245       
13246     }
13247 
13248   }
13249   
13250   /**
13251    * @param prompt
13252    */
13253   private void startLoader(boolean prompt) {
13254     
13255     boolean startLoader = true;
13256     
13257     if (prompt) {
13258       System.out.print("Do you want to start the Grouper loader (daemons)?\n  (note, if it is already running, you need to stop it now, check " 
13259           + (GrouperInstallerUtils.isWindows() ? "the task manager for java.exe" : "ps -ef | grep gsh | grep loader") + ") (t|f)? [f]: ");
13260       startLoader = readFromStdInBoolean(false, "grouperInstaller.autorun.startGrouperDaemons");
13261     }
13262     
13263     if (startLoader) {
13264       final List<String> commands = new ArrayList<String>();
13265       
13266       addGshCommands(commands);
13267       commands.add("-loader");
13268       
13269       if (!GrouperInstallerUtils.isWindows()) {
13270         
13271         //let this database run forever
13272         commands.add(0, "nohup");
13273         //run in new process
13274         commands.add("> /dev/null 2>&1 &");
13275         
13276         String fullCommand = GrouperInstallerUtils.join(commands.iterator(), ' ');
13277         commands.clear();
13278         commands.add(shCommand());
13279         commands.add("-c");
13280         commands.add(fullCommand);
13281         
13282       }
13283       System.out.println("\n##################################");
13284       System.out.println("Starting the Grouper loader (daemons): " + convertCommandsIntoCommand(commands) + "\n");
13285 
13286       //start in new thread
13287       Thread thread = new Thread(new Runnable() {
13288         
13289         @Override
13290         public void run() {
13291           GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
13292               true, true, null, GrouperInstaller.this.untarredApiDir, 
13293               GrouperInstaller.this.grouperInstallDirectoryString + "grouperLoader", false);
13294         }
13295       });
13296       thread.setDaemon(true);
13297       thread.start();
13298       
13299       System.out.println("\nEnd starting the Grouper loader (daemons)");
13300       System.out.println("##################################\n");
13301       
13302     }
13303 
13304   }
13305 
13306   /**
13307    * gsh command fully qualified
13308    */
13309   private String gshCommand;
13310   
13311   /**
13312    * 
13313    * @return the gsh command
13314    */
13315   private String gshCommand() {
13316 
13317     if (this.gshCommand == null) {
13318 
13319       String gshDir = GrouperInstallerUtils.defaultIfBlank(this.upgradeExistingApplicationDirectoryString, 
13320           this.untarredApiDir.getAbsolutePath() + File.separator);
13321       
13322       String gsh = gshDir + "bin" + File.separator 
13323           + (GrouperInstallerUtils.isWindows() ? "gsh.bat" : "gsh.sh");
13324       
13325       if (new File(gsh).exists()) {
13326         this.gshCommand = gsh;
13327         return gsh;
13328       }
13329 
13330       gsh = gshDir + "WEB-INF" + File.separator + "bin" + File.separator 
13331           + (GrouperInstallerUtils.isWindows() ? "gsh.bat" : "gsh.sh");
13332 
13333       if (new File(gsh).exists()) {
13334         this.gshCommand = gsh;
13335         return gsh;
13336       }
13337       
13338       throw new RuntimeException("Cant find gsh: " + gshDir);
13339     }
13340 
13341     return this.gshCommand;
13342   }
13343 
13344   /**
13345    * 
13346    */
13347   private void checkDatabaseConnection() {
13348     System.out.println("Checking database with query: " + this.giDbUtils.checkConnectionQuery());
13349     Exception exception = this.giDbUtils.checkConnection();
13350     if (exception == null) {
13351       System.out.println("Successfully tested database connection");
13352     } else {
13353       if (GrouperInstallerUtils.getFullStackTrace(exception).contains(ClassNotFoundException.class.getName())) {
13354         System.out.println("Cannot check connection since driver is not in classpath of installer, this is fine but not sure if connection details work or not");
13355       } else {
13356         System.out.println("Error: could not connect to the database: ");
13357         exception.printStackTrace();
13358       }
13359     }
13360   }
13361 
13362   /** gi db utils */
13363   private GiDbUtils giDbUtils = null;
13364   
13365   /**
13366    * 
13367    */
13368   private void configureTomcatGrouperUberWebapp(File tomcatDir, File webAppDir) {
13369     
13370     //GrouperInstallerUtils.toSet("catalina.sh", "startup.sh", "shutdown.sh");
13371     Set<String> shFileNames = new HashSet<String>();
13372 
13373     File binDir = new File(tomcatDir.getAbsolutePath() + File.separator + "bin");
13374 
13375     //get all sh files, doing wildcards doesnt work
13376     for (File file : binDir.listFiles()) {
13377       String fileName = GrouperInstallerUtils.defaultString(file.getName());
13378       if (file.isFile() && fileName.endsWith(".sh")) {
13379         shFileNames.add(fileName);
13380       }
13381     }
13382 
13383     for (String command : shFileNames) {
13384       List<String> commands = new ArrayList<String>();
13385       
13386       commands.add("chmod");
13387       commands.add("+x");
13388       //have to do * since all the  sh files need chmod
13389       commands.add(tomcatDir.getAbsolutePath() + File.separator + "bin" + File.separator + command);
13390 
13391       System.out.println("Making tomcat file executable with command: " + convertCommandsIntoCommand(commands) + "\n");
13392 
13393       CommandResult commandResult = GrouperInstallerUtils.execCommand(
13394           GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
13395           new File(tomcatDir.getAbsolutePath() + File.separator + "bin"), null, true);
13396       
13397       if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
13398         System.out.println("stderr: " + commandResult.getErrorText());
13399       }
13400       if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
13401         System.out.println("stdout: " + commandResult.getOutputText());
13402       }
13403     }
13404     
13405     Set<File> shFiles = new LinkedHashSet<File>();
13406     for (String shFileName : shFileNames) {
13407       shFiles.add(new File(shFileName));
13408     }
13409 // do this in container since its too dynamic
13410 //    for (String path : new String[] {"grouper", "grouper-ws"}) {
13411 //      // create grouper.xml in conf/Catalina/localhost/grouper.xml
13412 //      File tomeeGrouperFile = new File(tommeDir.getAbsolutePath() + File.separator + "conf" + File.separator +
13413 //          "Catalina" + File.separator + "localhost" + File.separator + path + ".xml");
13414 //      
13415 //      GrouperInstallerUtils.createParentDirectories(tomeeGrouperFile);
13416 //      GrouperInstallerUtils.fileCreate(tomeeGrouperFile);
13417 //      String cookiesFalse = "";
13418 //      if ("grouper-ws".equals(path)) {
13419 //        cookiesFalse = " cookies=\"false\" ";
13420 //      }
13421 //      String contentToWrite = "<Context docBase=\"/opt/grouper/grouperWebapp/\" path=\"/" + path + "\" reloadable=\"false\"" 
13422 //            + cookiesFalse + ">\n" + 
13423 //          "<Resources allowLinking=\"true\" />\n" + 
13424 //          "</Context>";
13425 //    
13426 //      try {      
13427 //        Files.write(Paths.get(tomeeGrouperFile.getAbsolutePath()), contentToWrite.toString().getBytes(), StandardOpenOption.APPEND);
13428 //      } catch (Exception e) {
13429 //        System.out.println("Could not write to grouper.xml file.");
13430 //      }
13431 //    }
13432     
13433 //    // edit server.xml in tomee/conf dir
13434 //    File serverXmlFile = new File(tommeDir.getAbsolutePath()
13435 //        + File.separator + "conf" + File.separator + "server.xml");
13436 //    
13437 //    Map<String, String> expectedAttribute = new HashMap<String, String>();
13438 //
13439 //    expectedAttribute.put("port", "8009");
13440 //    expectedAttribute.put("protocol", "AJP/1.3");
13441 //
13442 //    editXmlFileAttribute(serverXmlFile, "Connector", expectedAttribute, "tomcatAuthentication", "false", 
13443 //         "Set tomcatAuthentication to false");
13444 //    editXmlFileAttribute(serverXmlFile, "Connector", expectedAttribute, "URIEncoding", "UTF-8", 
13445 //        "Set URIEncoding to UTF-8");
13446 //    editXmlFileAttribute(serverXmlFile, "Connector", expectedAttribute, "scheme", "https", 
13447 //        "Set scheme to https");
13448 //    editXmlFileAttribute(serverXmlFile, "Connector", expectedAttribute, "secure", "true", 
13449 //        "Set secure to true");
13450     
13451   }
13452   
13453 
13454   /**
13455    * 
13456    */
13457   private void configureTomcatUiWebapp() {
13458     
13459     File serverXmlFile = new File(this.untarredTomcatDir.getAbsolutePath() 
13460         + File.separator + "conf" + File.separator + "server.xml");
13461     
13462     //C:\mchyzer\grouper\trunk\grouper-installer\grouper.ui-2.0.2\dist\grouper
13463     //
13464     //<Context docBase="C:\mchyzer\grouper\trunk\grouper-ws_trunk\webapp" path="/grouper-ws" reloadable="false"/>
13465     //Server
13466     //Service
13467     //Engine
13468     //Host
13469 
13470     System.out.print("Enter the URL path for the UI [grouper]: ");
13471     this.tomcatUiPath = readFromStdIn("grouperInstaller.autorun.urlPathForUi");
13472     
13473     if (GrouperInstallerUtils.isBlank(this.tomcatUiPath)) {
13474       this.tomcatUiPath = "grouper";
13475     }
13476 
13477     if (this.tomcatUiPath.endsWith("/") || this.tomcatUiPath.endsWith("\\")) {
13478       this.tomcatUiPath = this.tomcatUiPath.substring(0, this.tomcatUiPath.length()-1);
13479     }
13480     if (this.tomcatUiPath.startsWith("/") || this.tomcatUiPath.startsWith("\\")) {
13481       this.tomcatUiPath = this.tomcatUiPath.substring(1, this.tomcatUiPath.length());
13482     }
13483     
13484     String currentDocBase = GrouperInstallerUtils.xpathEvaluateAttribute(serverXmlFile, 
13485         "Server/Service/Engine/Host/Context[@path='/" + this.tomcatUiPath + "']", "docBase");
13486 
13487     String shouldBeDocBase = grouperUiBuildToDirName();
13488 
13489     System.out.println("Editing tomcat config file: " + serverXmlFile.getAbsolutePath());
13490     
13491     if (GrouperInstallerUtils.isBlank(currentDocBase)) {
13492 
13493       //need to add it
13494       //<Host appBase="webapps" autoDeploy="true" name="localhost" unpackWARs="true" xmlNamespaceAware="false" xmlValidation="false">
13495       //<Context docBase="C:\mchyzer\grouper\trunk\grouper-ws_trunk\webapp" path="/grouper-ws" reloadable="false"/>
13496       addToXmlFile(serverXmlFile, ">", new String[]{"<Host "}, 
13497           "<Context docBase=\"" + shouldBeDocBase + "\" path=\"/" + this.tomcatUiPath + "\" reloadable=\"false\"/>", "tomcat context for UI");
13498 
13499     } else {
13500 
13501       if (!GrouperInstallerUtils.equals(currentDocBase, shouldBeDocBase)) {
13502         
13503         //lets edit the file
13504         //<Context docBase="C:\mchyzer\grouper\trunk\grouper-ws_trunk\webapp" path="/grouper-ws" reloadable="false"/>
13505         editFile(serverXmlFile, "docBase=\"([^\"]+)\"", new String[]{"<Context", "path=\"/" + this.tomcatUiPath + "\""}, 
13506             null, shouldBeDocBase, "tomcat context for UI");
13507 
13508       } else {
13509         
13510         System.out.println("  - Context is already set for Grouper UI");
13511         
13512       }
13513       
13514       
13515     }
13516     
13517     currentDocBase = GrouperInstallerUtils.xpathEvaluateAttribute(serverXmlFile, 
13518         "Server/Service/Engine/Host/Context[@path='/" + this.tomcatUiPath + "']", "docBase");
13519     
13520     if (!GrouperInstallerUtils.equals(currentDocBase, shouldBeDocBase)) {
13521       System.out.println("Tried to edit server.xml but it didnt work, should have context of: '" 
13522           + shouldBeDocBase + "', but was: '" + currentDocBase + "'");
13523     }
13524     
13525   }
13526 
13527   /**
13528    * 
13529    * @return grouper ui build to name
13530    */
13531   private String grouperUiBuildToDirName() {
13532     return this.untarredUiDir.getAbsolutePath() + File.separator + "dist" + File.separator + "grouper";
13533   }
13534 
13535   /**
13536    * @param isInstallNotUpgrade
13537    */
13538   private void buildWs(boolean isInstallNotUpgrade) {
13539     
13540     File grouperWsBuildToDir = new File(this.grouperWsBuildToDirName());
13541     
13542     if (grouperWsBuildToDir.exists()) {
13543 
13544       boolean rebuildWs = true;
13545       
13546       boolean defaultRebuild = GrouperInstallerUtils.propertiesValueBoolean("grouperInstaller.default.ws.rebuildIfBuilt", true, false);
13547       System.out.print("The Grouper WS has been built in the past, do you want it rebuilt? (t|f) [" 
13548           + (defaultRebuild ? "t" : "f") + "]: ");
13549       rebuildWs = readFromStdInBoolean(defaultRebuild, "grouperInstaller.autorun.rebuildWsIfBuiltAlready");
13550   
13551       if (!rebuildWs) {
13552         return;
13553       }
13554     }
13555     
13556     if (isInstallNotUpgrade) {
13557       //stop tomcat
13558       try {
13559         tomcatBounce("stop");
13560       } catch (Throwable e) {
13561         System.out.println("Couldnt stop tomcat, ignoring...");
13562       }
13563     }
13564     
13565     List<String> commands = new ArrayList<String>();
13566     
13567     addAntCommands(commands);
13568     commands.add("dist");
13569     
13570     System.out.println("\n##################################");
13571     System.out.println("Building WS with command:\n" + this.untarredWsDir.getAbsolutePath() + File.separator + "grouper-ws" + "> " 
13572         + convertCommandsIntoCommand(commands) + "\n");
13573     
13574     CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
13575         true, true, null, new File(this.untarredWsDir.getAbsolutePath() + File.separator + "grouper-ws"), null, true);
13576     
13577     if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
13578       System.out.println("stderr: " + commandResult.getErrorText());
13579     }
13580     if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
13581       System.out.println("stdout: " + commandResult.getOutputText());
13582     }
13583 
13584     if (isInstallNotUpgrade) {
13585       System.out.print("Do you want to set the log dir of WS (t|f)? [t]: ");
13586       boolean setLogDir = readFromStdInBoolean(true, "grouperInstaller.autorun.setWsLogDir");
13587       
13588       if (setLogDir) {
13589         
13590         ////set the log dir
13591         //C:\apps\grouperInstallerTest\grouper.ws-2.0.2\grouper-ws\build\dist\grouper-ws\WEB-INF\classes\log4j.properties
13592         //
13593         //${grouper.home}logs
13594   
13595         String defaultLogDir = this.untarredTomcatDir.getAbsolutePath() + File.separator + "logs" + File.separator + "grouperWs";
13596         System.out.print("Enter the WS log dir: [" + defaultLogDir + "]: ");
13597         String logDir = readFromStdIn("grouperInstaller.autorun.wsLogDir");
13598         logDir = GrouperInstallerUtils.defaultIfBlank(logDir, defaultLogDir);
13599         
13600         //lets replace \\ with /
13601         logDir = GrouperInstallerUtils.replace(logDir, "\\\\", "/");
13602         //lets replace \ with /
13603         logDir = GrouperInstallerUtils.replace(logDir, "\\", "/");
13604   
13605         File log4jFile = new File(grouperWsBuildToDirName() + File.separator + "WEB-INF" + File.separator + "classes"
13606             + File.separator + "log4j.properties");
13607         
13608         System.out.println("Editing file: " + log4jFile.getAbsolutePath());
13609         
13610         editFile(log4jFile, "log4j\\.\\S+\\.File\\s*=\\s*([^\\s]+logs)/grouper_[^\\s]+\\.log", null, 
13611             null, logDir, "WS log directory");
13612         
13613         File logDirFile = new File(defaultLogDir);
13614         if (!logDirFile.exists()) {
13615           System.out.println("Creating log directory: " + logDirFile.getAbsolutePath());
13616           GrouperInstallerUtils.mkdirs(logDirFile);
13617         }
13618         //test log dir
13619         File testLogDirFile = new File(logDirFile.getAbsolutePath() + File.separator + "testFile" + GrouperInstallerUtils.uniqueId() + ".txt");
13620         GrouperInstallerUtils.saveStringIntoFile(testLogDirFile, "test");
13621         if (!testLogDirFile.delete()) {
13622           throw new RuntimeException("Cant delete file: " +  testLogDirFile.getAbsolutePath());
13623         }
13624         System.out.println("Created and deleted a test file successfully in dir: " + logDirFile.getAbsolutePath());
13625       }
13626     }
13627     
13628     System.out.println("\nEnd building Ws");
13629     System.out.println("##################################\n");
13630   
13631     
13632   }
13633 
13634   /**
13635    * 
13636    */
13637   private void configureTomcatWsWebapp() {
13638     
13639     File serverXmlFile = new File(this.untarredTomcatDir.getAbsolutePath() 
13640         + File.separator + "conf" + File.separator + "server.xml");
13641     
13642     //C:\mchyzer\grouper\trunk\grouper-installer\grouper.ui-2.0.2\dist\grouper
13643     //
13644     //<Context docBase="C:\mchyzer\grouper\trunk\grouper-ws_trunk\webapp" path="/grouper-ws" reloadable="false"/>
13645     //Server
13646     //Service
13647     //Engine
13648     //Host
13649   
13650     System.out.print("Enter the URL path for the WS [grouper-ws]: ");
13651     this.tomcatWsPath = readFromStdIn("grouperInstaller.autorun.wsUrlPath");
13652     
13653     if (GrouperInstallerUtils.isBlank(this.tomcatWsPath)) {
13654       this.tomcatWsPath = "grouper-ws";
13655     }
13656   
13657     if (this.tomcatWsPath.endsWith("/") || this.tomcatWsPath.endsWith("\\")) {
13658       this.tomcatWsPath = this.tomcatWsPath.substring(0, this.tomcatWsPath.length()-1);
13659     }
13660     if (this.tomcatWsPath.startsWith("/") || this.tomcatWsPath.startsWith("\\")) {
13661       this.tomcatWsPath = this.tomcatWsPath.substring(1);
13662     }
13663     
13664     String currentDocBase = GrouperInstallerUtils.xpathEvaluateAttribute(serverXmlFile, 
13665         "Server/Service/Engine/Host/Context[@path='/" + this.tomcatWsPath + "']", "docBase");
13666   
13667     //grouper.ws-2.0.2\grouper-ws\build\dist\grouper-ws
13668     
13669     String shouldBeDocBase = grouperWsBuildToDirName();
13670   
13671     System.out.println("Editing tomcat config file: " + serverXmlFile.getAbsolutePath());
13672     
13673     if (GrouperInstallerUtils.isBlank(currentDocBase)) {
13674   
13675       //need to add it
13676       //<Host appBase="webapps" autoDeploy="true" name="localhost" unpackWARs="true" xmlNamespaceAware="false" xmlValidation="false">
13677       //<Context docBase="C:\mchyzer\grouper\trunk\grouper-ws_trunk\webapp" path="/grouper-ws" reloadable="false"/>
13678       addToXmlFile(serverXmlFile, ">", new String[]{"<Host "}, 
13679           "<Context docBase=\"" + shouldBeDocBase + "\" path=\"/" + this.tomcatWsPath + "\" reloadable=\"false\"/>", "tomcat context for WS");
13680   
13681     } else {
13682   
13683       if (!GrouperInstallerUtils.equals(currentDocBase, shouldBeDocBase)) {
13684         
13685         //lets edit the file
13686         //<Context docBase="C:\mchyzer\grouper\trunk\grouper-ws_trunk\webapp" path="/grouper-ws" reloadable="false"/>
13687         editFile(serverXmlFile, "docBase=\"([^\"]+)\"", new String[]{"<Context", "path=\"/" + this.tomcatWsPath + "\""}, 
13688             null, shouldBeDocBase, "tomcat context for WS");
13689   
13690       } else {
13691         
13692         System.out.println("  - Context is already set for Grouper WS");
13693         
13694       }
13695       
13696       
13697     }
13698     
13699     currentDocBase = GrouperInstallerUtils.xpathEvaluateAttribute(serverXmlFile, 
13700         "Server/Service/Engine/Host/Context[@path='/" + this.tomcatWsPath + "']", "docBase");
13701     
13702     if (!GrouperInstallerUtils.equals(currentDocBase, shouldBeDocBase)) {
13703       System.out.println("Tried to edit server.xml but it didnt work, should have context of: '" 
13704           + shouldBeDocBase + "', but was: '" + currentDocBase + "'");
13705     }
13706     
13707   }
13708 
13709   /**
13710    * @return grouper ws build to dir name
13711    */
13712   private String grouperWsBuildToDirName() {
13713     return this.untarredWsDir.getAbsolutePath() + File.separator + "grouper-ws" + File.separator 
13714       + "build" + File.separator + "dist" + File.separator + "grouper-ws";
13715   }
13716   
13717   /**
13718      * 
13719      */
13720     private void configureClient() {
13721       //properties file
13722       File localGrouperClientPropertiesFile = new File(this.untarredClientDir.getAbsolutePath() + File.separator 
13723           + "grouper.client.properties");
13724       
13725       //set the grouper property
13726       System.out.println("Editing " + localGrouperClientPropertiesFile.getAbsolutePath() + ": ");
13727       editPropertiesFile(localGrouperClientPropertiesFile, "grouperClient.webService.url", "http://localhost:" 
13728           + this.tomcatHttpPort + "/" + this.tomcatWsPath + "/servicesRest", false);
13729       editPropertiesFile(localGrouperClientPropertiesFile, "grouperClient.webService.login", "GrouperSystem", false);
13730       editPropertiesFile(localGrouperClientPropertiesFile, "grouperClient.webService.password", this.grouperSystemPassword, false);
13731       
13732       
13733 //      grouperClient.webService.url = http://localhost:8200/grouper-ws/servicesRest
13734 //
13735 //      # kerberos principal used to connect to web service
13736 //      grouperClient.webService.login = GrouperSystem
13737 //
13738 //      # password for shared secret authentication to web service
13739 //      # or you can put a filename with an encrypted password
13740 //      grouperClient.webService.password = myNewPass
13741 
13742       
13743     }
13744 
13745   /**
13746    * 
13747    * @return the file of the directory of WS
13748    */
13749   private File downloadClient() {
13750     String urlToDownload = GrouperInstallerUtils.propertiesValue("download.server.url", true);
13751     
13752     if (!urlToDownload.endsWith("/")) {
13753       urlToDownload += "/";
13754     }
13755     urlToDownload += "release/";
13756   
13757     String clientFileName = "grouper.clientBinary-" + this.version + ".tar.gz";
13758     urlToDownload += this.version + "/" + clientFileName;
13759   
13760     File clientFile = new File(this.grouperTarballDirectoryString + clientFileName);
13761     
13762     downloadFile(urlToDownload, clientFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalClientDownloadTarEtc");
13763 
13764     return clientFile;
13765   }
13766 
13767   /**
13768    * 
13769    */
13770   private void addGrouperSystemWsGroup() {
13771 
13772     //C:\mchyzer\grouper\trunk\grouper-installer\grouper.apiBinary-2.0.2\bin>gsh -runarg "grouperSession = GrouperSession.startRootSession();\nwsGroup = new GroupSave(grouperSession).assignName(\"etc:webServiceClientUsers\").assignCreateParentStemsIfNotExist(true).save();\nwsGroup.addMember(SubjectFinder.findRootSubject(), false);"
13773 
13774     //running with command on command line doenst work on linux since the args with whitespace translate to 
13775     //save the commands to a file, and runt he file
13776     StringBuilder gshCommands = new StringBuilder();
13777     gshCommands.append("grouperSession = GrouperSession.startRootSession();\n");
13778     gshCommands.append("wsGroup = new GroupSave(grouperSession).assignName(\"etc:webServiceClientUsers\").assignCreateParentStemsIfNotExist(true).save();\n");
13779     gshCommands.append("wsGroup.addMember(SubjectFinder.findRootSubject(), false);\n");
13780     
13781     File gshFile = new File(this.untarredApiDir.getAbsolutePath() + File.separator + "gshAddGrouperSystemWsGroup.gsh");
13782     GrouperInstallerUtils.saveStringIntoFile(gshFile, gshCommands.toString());
13783     
13784     List<String> commands = new ArrayList<String>();
13785 
13786     addGshCommands(commands);
13787     commands.add(gshFile.getAbsolutePath());
13788 
13789     System.out.println("\n##################################");
13790     System.out.println("Adding user GrouperSystem to grouper-ws users group with command:\n  " + convertCommandsIntoCommand(commands) + "\n");
13791 
13792     CommandResult commandResult = GrouperInstallerUtils.execCommand(
13793         GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
13794         this.untarredApiDir, null, true);
13795 
13796     if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
13797       System.out.println("stderr: " + commandResult.getErrorText());
13798     }
13799     if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
13800       System.out.println("stdout: " + commandResult.getOutputText());
13801     }
13802 
13803 
13804   }
13805 
13806   /**
13807    * 
13808    */
13809   private void runChangeLogTempToChangeLog() {
13810 
13811     boolean defaultBoolean = GrouperInstallerUtils.propertiesValueBoolean("grouperInstaller.default.api.runChangeLogToChangeLogTemp", true, false);
13812     System.out.print("Is it ok to run a script that copies change log temp records to the change log (recommended) (t|f)? [" 
13813         + (defaultBoolean ? "t" : "f") + "]: ");
13814     boolean runScript = readFromStdInBoolean(defaultBoolean, "grouperInstaller.autorun.runChangeLogTempToChangeLog");
13815 
13816     
13817     if (!runScript) {
13818       return;
13819     }
13820     
13821     //running with command on command line doenst work on linux since the args with whitespace translate to 
13822     //save the commands to a file, and runt he file
13823     StringBuilder gshCommands = new StringBuilder();
13824     gshCommands.append("grouperSession = GrouperSession.startRootSession();\n");
13825     gshCommands.append("loaderRunOneJob(\"CHANGE_LOG_changeLogTempToChangeLog\");\n");
13826     
13827     File gshFile = new File(this.untarredApiDir.getAbsolutePath() + File.separator + "gshChangeLogTempToChangeLog.gsh");
13828     GrouperInstallerUtils.saveStringIntoFile(gshFile, gshCommands.toString());
13829     
13830     List<String> commands = new ArrayList<String>();
13831 
13832     addGshCommands(commands);
13833     commands.add(gshFile.getAbsolutePath());
13834 
13835     System.out.println("\n##################################");
13836     System.out.println("Copying records from change log temp to change log with command:\n  " + convertCommandsIntoCommand(commands) + "\n");
13837 
13838     CommandResult commandResult = GrouperInstallerUtils.execCommand(
13839         GrouperInstallerUtils.toArray(commands, String.class), true, true, null, 
13840        new File(this.gshCommand()).getParentFile(), null, true);
13841 
13842     if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
13843       System.out.println("stderr: " + commandResult.getErrorText());
13844     }
13845     if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
13846       System.out.println("stdout: " + commandResult.getOutputText());
13847     }
13848 
13849 
13850   }
13851 
13852   /**
13853    * 
13854    */
13855   private void runClientCommand() {
13856     System.out.println("##################################");
13857     System.out.println("Running client command:");
13858     System.out.println(this.untarredClientDir.getAbsolutePath() + "> " + getJavaCommand()
13859         + " -jar grouperClient.jar --operation=getMembersWs --groupNames=etc:webServiceClientUsers");
13860     
13861     try {
13862       final List<String> command = new ArrayList<String>();
13863   
13864       command.add(getJavaCommand());
13865       command.add("-jar");
13866       command.addAll(GrouperInstallerUtils.splitTrimToList(
13867           "grouperClient.jar --operation=getMembersWs --groupNames=etc:webServiceClientUsers", " "));
13868               
13869       CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(command, String.class), 
13870           true, true, null, this.untarredClientDir, null, true);
13871   
13872       if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
13873         System.out.println("stderr: " + commandResult.getErrorText());
13874       }
13875       if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
13876         System.out.println("stdout: " + commandResult.getOutputText());
13877       }
13878       System.out.println("Success running client command:");
13879     } catch (Exception e) {
13880       System.out.println("Exception running Grouper client");
13881       e.printStackTrace();
13882       System.out.print("Press <enter> to continue: ");
13883       readFromStdIn("grouperInstaller.autorun.grouperClientErrorContinue");
13884     }
13885     System.out.println("##################################");
13886 
13887   }
13888 
13889   /**
13890    * edit a property in a property file
13891    * @param file
13892    * @param valueRegex 
13893    * @param lineMustHaveRegexes 
13894    * @param lineCantHaveRegexes 
13895    * @param newValue 
13896    * @param description of change for sys out print
13897    * @return true if edited file, or false if not but didnt need to, null if not found
13898    */
13899   public static Boolean editFile(File file, String valueRegex, String[] lineMustHaveRegexes, 
13900       String[] lineCantHaveRegexes, String newValue, String description) {
13901     return editFile(file, valueRegex, lineMustHaveRegexes, lineCantHaveRegexes, newValue, description, false, null);
13902   }
13903 
13904   /**
13905    * edit a property in a property file
13906    * @param file
13907    * @param valueRegex 
13908    * @param lineMustHaveRegexes 
13909    * @param lineCantHaveRegexes 
13910    * @param newValue 
13911    * @param description of change for sys out print
13912    * @param addAttributeIfNotExists if attribute isnt there, then if true, then add the attribute
13913    * @param newAttributeName if adding new attribute, this is the name
13914    * @return true if edited file, or false if not but didnt need to, null if not found
13915    */
13916   public static Boolean editFile(File file, String valueRegex, String[] lineMustHaveRegexes, 
13917       String[] lineCantHaveRegexes, String newValue, String description, boolean addAttributeIfNotExists, String newAttributeName) {
13918     
13919     if (!GrouperInstallerUtils.isBlank(newAttributeName) != addAttributeIfNotExists) {
13920       throw new RuntimeException("newAttributeName cant be null if addAttributeIfNotExists, and must be null if not addAttributeIfNotExists");
13921     }
13922     
13923     if (!file.exists() || file.length() == 0) {
13924       throw new RuntimeException("Why does " + file.getName() + " not exist and have contents? " 
13925           + file.getAbsolutePath());
13926     }
13927     
13928     String fileContents = GrouperInstallerUtils.readFileIntoString(file);
13929     
13930     String newline = GrouperInstallerUtils.newlineFromFile(fileContents);
13931     
13932     String[] lines = GrouperInstallerUtils.splitLines(fileContents);
13933     
13934     Pattern pattern = Pattern.compile(valueRegex);
13935     
13936     Pattern[] lineMustHavePatterns = new Pattern[GrouperInstallerUtils.length(lineMustHaveRegexes)];
13937     
13938     {
13939       int index = 0;
13940       for (String lineMustHaveRegex : GrouperInstallerUtils.nonNull(lineMustHaveRegexes, String.class)) {
13941         Pattern lineMustHavePattern = Pattern.compile(lineMustHaveRegex);
13942         lineMustHavePatterns[index] = lineMustHavePattern;
13943         
13944         index++;
13945       }
13946     }    
13947   
13948     Pattern[] lineCantHavePatterns = new Pattern[GrouperInstallerUtils.length(lineCantHaveRegexes)];
13949     
13950     {
13951       int index = 0;
13952       for (String lineCantHaveRegex : GrouperInstallerUtils.nonNull(lineCantHaveRegexes, String.class)) {
13953         Pattern lineCantHavePattern = Pattern.compile(lineCantHaveRegex);
13954         lineCantHavePatterns[index] = lineCantHavePattern;
13955         
13956         index++;
13957       }
13958     }    
13959   
13960     StringBuilder newfile = new StringBuilder();
13961     
13962     boolean madeChange = false;
13963     boolean noChangeNeeded = false;
13964     
13965     OUTER: for (String line : lines) {
13966       line = GrouperInstallerUtils.defaultString(line);
13967       
13968       //lets see if it satisfies all
13969       for (Pattern lineMustHavePattern : lineMustHavePatterns) {
13970         if (!lineMustHavePattern.matcher(line).find()) {
13971           newfile.append(line).append(newline);
13972           continue OUTER;
13973         }
13974       }
13975       
13976       //lets see if it doesnt have these
13977       for (Pattern lineCantHavePattern : lineCantHavePatterns) {
13978         if (lineCantHavePattern.matcher(line).find()) {
13979           newfile.append(line).append(newline);
13980           continue OUTER;
13981         }
13982       }
13983       
13984       //see if satisfies current
13985       Matcher matcher = pattern.matcher(line);
13986       if (!matcher.find()) {
13987         
13988         if (addAttributeIfNotExists) {
13989           
13990           System.out.println(" - adding " + description + " with value: '" + newValue + "'");
13991           
13992           line = GrouperInstallerUtils.trimEnd(line);
13993           
13994           boolean endsWithCloseTag = false;
13995           boolean endElement = false;
13996           
13997           if (line.endsWith("/>")) {
13998             line = line.substring(0, line.length()-2);
13999             line = GrouperInstallerUtils.trimEnd(line);
14000             endsWithCloseTag = true;
14001           } else if (line.endsWith(">")) {
14002             line = line.substring(0, line.length()-1);
14003             line = GrouperInstallerUtils.trimEnd(line);
14004             endElement = true;
14005           }
14006           
14007           newfile.append(line).append(" ").append(newAttributeName).append("=\"").append(newValue).append("\"");
14008           
14009           if (endsWithCloseTag) {
14010             newfile.append(" />");
14011           } else if (endElement) {
14012             newfile.append(" >");
14013           }
14014           
14015           newfile.append(newline);
14016           madeChange = true;
14017           
14018         } else {
14019         
14020           newfile.append(line).append(newline);
14021         }
14022         continue;
14023       }
14024       
14025       String oldValue = matcher.group(1);
14026       if (GrouperInstallerUtils.equals(newValue, oldValue)) {
14027         System.out.println(" - old " + description + " value is same as new value: " + newValue);
14028         noChangeNeeded = true;
14029         newfile.append(line).append(newline);
14030         continue;
14031       }
14032       
14033       //we need to change the value
14034       System.out.println(" - changing " + description + " from: '" + oldValue + "' to: '" + newValue + "'");
14035       newfile.append(line.substring(0, matcher.start(1)));
14036       newfile.append(newValue);
14037       newfile.append(line.substring(matcher.end(1), line.length()));
14038       newfile.append(newline);
14039       madeChange = true;
14040       continue;
14041     }
14042     
14043     if (!madeChange) {
14044       //true if edited file, or false if not but didnt need to, null if not found
14045       if (noChangeNeeded) {
14046         return false;
14047       }
14048       return null;
14049     }
14050     
14051     GrouperInstallerUtils.writeStringToFile(file, newfile.toString());
14052     
14053     return true;
14054   }
14055 
14056   /**
14057    * add a line to a file.  will replace \n with whatever newline is
14058    * @param file
14059    * @param line (not ending in newline)
14060    * @param lineNumber 1 indexed.  If not exist, add to end of file
14061    * @param description is a description of what was just changed
14062    */
14063   public static void addToFile(File file, String line, int lineNumber, String description) {
14064     if (!file.exists() || file.length() == 0) {
14065       throw new RuntimeException("Why does " + file.getName() + " not exist and have contents? " 
14066           + file.getAbsolutePath());
14067     }
14068     
14069     String fileContents = GrouperInstallerUtils.readFileIntoString(file);
14070     
14071     String newline = GrouperInstallerUtils.newlineFromFile(fileContents);
14072     
14073     String[] lines = GrouperInstallerUtils.splitLines(fileContents);
14074 
14075     line = GrouperInstallerUtils.replace(line, "\n", newline);
14076     
14077     line += newline;
14078     
14079     StringBuilder newfile = new StringBuilder();
14080     
14081     boolean madeChange = false;
14082     
14083     int index = 0;
14084     
14085     for (String fileLine : lines) {
14086       fileLine = GrouperInstallerUtils.defaultString(fileLine);
14087       newfile.append(fileLine).append(newline);
14088       index++;
14089       
14090       if (index >= lineNumber  && !madeChange) {
14091 
14092         System.out.println("Adding " + description + " to file at line number: " + lineNumber);        
14093         
14094         newfile.append(line);
14095         madeChange = true;
14096       }
14097     }
14098     
14099     if (!madeChange) {
14100       System.out.println("Appending " + description + " to end of file");        
14101       newfile.append(line);
14102     }
14103     
14104     GrouperInstallerUtils.writeStringToFile(file, newfile.toString());
14105     
14106   }
14107 
14108   /** tomcat ui path */
14109   private String tomcatUiPath = null;
14110 
14111   /** tomcat ws path */
14112   private String tomcatWsPath = null;
14113 
14114   /** untarred dir, this does NOT end in file.separator */
14115   private File untarredClientDir;
14116 
14117   /**
14118    * see if install, upgrade, patch, etc
14119    * @return true if install, or false if upgrade
14120    */
14121   private GrouperInstallerMainFunction grouperInstallerMainFunction() {
14122 
14123     GrouperInstallerMainFunction grouperInstallerMainFunctionLocal = 
14124         (GrouperInstallerMainFunction)promptForEnum(
14125             "Do you want to install ('installContainer') a new grouper container , 'upgrade' an existing installation,\n"
14126                 + "  'patch' an existing installation, 'admin' utilities, 'buildContainer', 'installContainer', or 'createPatch' for Grouper developers\n" 
14127                 + "  (enter: 'installContainer', 'upgrade', 'patch', 'admin', 'createPatch', 'buildContainer', or blank for the default) ",
14128             "grouperInstaller.autorun.actionEgInstallUpgradePatch", GrouperInstallerMainFunction.class, 
14129             GrouperInstallerMainFunction.installContainer, "grouperInstaller.default.installOrUpgrade");
14130     return grouperInstallerMainFunctionLocal;
14131   }
14132 
14133 
14134   
14135   /**
14136    * 
14137    * @return install directory
14138    */
14139   private static String grouperInstallDirectory() {
14140     String grouperInstallDirectoryString;
14141     {
14142       File grouperInstallDirectoryFile = new File("");
14143       String defaultDirectory = GrouperInstallerUtils.propertiesValue("grouperInstaller.default.installDirectory", false);
14144       System.out.print("Enter in the Grouper install directory (note: better if no spaces or special chars) [" 
14145           + (GrouperInstallerUtils.isBlank(defaultDirectory) ? grouperInstallDirectoryFile.getAbsolutePath() : defaultDirectory) + "]: ");
14146       String input = readFromStdIn("grouperInstaller.autorun.installDirectory");
14147       if (!GrouperInstallerUtils.isBlank(input)) {
14148         grouperInstallDirectoryFile = new File(input);
14149         if (!grouperInstallDirectoryFile.exists() || !grouperInstallDirectoryFile.isDirectory()) {
14150           System.out.println("Error: cant find directory: '" + input + "'");
14151           System.exit(1);
14152         }
14153       } else {
14154         if (!GrouperInstallerUtils.isBlank(defaultDirectory)) {
14155           grouperInstallDirectoryFile = new File(defaultDirectory);
14156           if (!grouperInstallDirectoryFile.exists() || !grouperInstallDirectoryFile.isDirectory()) {
14157             System.out.println("Error: cant find directory: '" + input + "'");
14158             System.exit(1);
14159           }
14160         }
14161       }
14162       grouperInstallDirectoryString = grouperInstallDirectoryFile.getAbsolutePath();
14163       if (!grouperInstallDirectoryString.endsWith(File.separator)) {
14164         grouperInstallDirectoryString += File.separator;
14165       }
14166     }
14167     return grouperInstallDirectoryString;
14168   }
14169   
14170   /**
14171    * 
14172    * @return upgrade directory
14173    */
14174   private String grouperUpgradeTempDirectory() {
14175     String localGrouperInstallDirectoryString = null;
14176     {
14177       File grouperInstallDirectoryFile = new File(new File("").getAbsolutePath() + File.separator + "tarballs");
14178       if (!GrouperInstallerUtils.isBlank(this.grouperInstallDirectoryString)) {
14179         grouperInstallDirectoryFile = new File(this.grouperInstallDirectoryString + "tarballs");
14180       }
14181       String defaultDirectory = GrouperInstallerUtils.propertiesValue("grouperInstaller.default.tarballDirectory", false);
14182       if (GrouperInstallerUtils.isBlank(defaultDirectory)) {
14183         defaultDirectory = grouperInstallDirectoryFile.getAbsolutePath();
14184       }
14185       System.out.print("Enter in a Grouper temp directory to download tarballs (note: better if no spaces or special chars) [" 
14186           + defaultDirectory + "]: ");
14187       localGrouperInstallDirectoryString = readFromStdIn("grouperInstaller.autorun.tarballDirectory");
14188       if (!GrouperInstallerUtils.isBlank(localGrouperInstallDirectoryString)) {
14189         grouperInstallDirectoryFile = new File(localGrouperInstallDirectoryString);
14190         if (!grouperInstallDirectoryFile.exists() || !grouperInstallDirectoryFile.isDirectory()) {
14191           System.out.println("Error: cant find directory: '" + grouperInstallDirectoryFile.getAbsolutePath() + "'");
14192           System.exit(1);
14193         }
14194       } else {
14195         localGrouperInstallDirectoryString = defaultDirectory;
14196       }
14197       if (!localGrouperInstallDirectoryString.endsWith(File.separator)) {
14198         localGrouperInstallDirectoryString += File.separator;
14199       }
14200     }
14201     return localGrouperInstallDirectoryString;
14202   }
14203   
14204   /**
14205    * 
14206    * @return grouper container directory
14207    */
14208   private String grouperContainerDirectory() {
14209     String localGrouperContainerDirectoryString = null;
14210     {
14211       File grouperContainerDirectoryFile = new File(new File("").getAbsolutePath() + File.separator + "container");
14212       if (!GrouperInstallerUtils.isBlank(this.grouperInstallDirectoryString)) {
14213         grouperContainerDirectoryFile = new File(this.grouperInstallDirectoryString + "container");
14214       }
14215       String defaultDirectory = GrouperInstallerUtils.propertiesValue("grouperInstaller.default.buildContainerDirectory", false);
14216       if (GrouperInstallerUtils.isBlank(defaultDirectory)) {
14217         defaultDirectory = grouperContainerDirectoryFile.getAbsolutePath();
14218       }
14219       System.out.print("Enter in a directory for output (note: better if no spaces or special chars) [" 
14220           + defaultDirectory + "]: ");
14221       localGrouperContainerDirectoryString = readFromStdIn("grouperInstaller.autorun.buildContainerDirectory");
14222       if (!GrouperInstallerUtils.isBlank(localGrouperContainerDirectoryString)) {
14223         grouperContainerDirectoryFile = new File(localGrouperContainerDirectoryString);
14224         if (!grouperContainerDirectoryFile.exists() || !grouperContainerDirectoryFile.isDirectory()) {
14225           System.out.println("Error: cant find directory: '" + grouperContainerDirectoryFile.getAbsolutePath() + "'");
14226           System.exit(1);
14227         }
14228       } else {
14229         localGrouperContainerDirectoryString = defaultDirectory;
14230       }
14231       if (!localGrouperContainerDirectoryString.endsWith(File.separator)) {
14232         localGrouperContainerDirectoryString += File.separator;
14233       }
14234     }
14235     return localGrouperContainerDirectoryString;
14236   }
14237   
14238   /**
14239    * 
14240    * @return see if operating on a source directory or deployed directory
14241    */
14242   @SuppressWarnings("unused")
14243   private GrouperInstallType sourceOrDeployed() {
14244 
14245     if (this.grouperDirectories.getGrouperInstallType() == null) {
14246              
14247     }
14248     
14249     return this.grouperDirectories.getGrouperInstallType();
14250   }
14251 
14252 
14253   
14254   /**
14255    * 
14256    * @return directory where existing installation exists
14257    */
14258   private String upgradeExistingDirectory() {
14259 
14260     //get the directory where the existing installation is
14261     String tempUpgradeExistingApplicationDirectoryString = this.upgradeExistingApplicationDirectoryString;
14262 
14263     String errorMessage = "Cant find Grouper " + this.appToUpgrade.name() + " properties files or libs, looked in the directory, "
14264         + "/classes/ , /conf/ , /WEB-INF/classes/ , /lib/ , /WEB-INF/lib/ , /lib/grouper/ , /dist/lib/ ";
14265 
14266     try {
14267       String upgradeExistingDirectoryString = null;
14268       for (int i=0;i<10;i++) {
14269         File grouperInstallDirectoryFile = new File("");
14270         String defaultDirectory = GrouperInstallerUtils.propertiesValue("grouperInstaller.default.existingInstalledDirectory", false);
14271         System.out.print("Where is the grouper " + this.appToUpgrade.name() + " installed? " +
14272           (GrouperInstallerUtils.isBlank(defaultDirectory) ? "" : ("[" + defaultDirectory + "]: ")));
14273         upgradeExistingDirectoryString = readFromStdIn("grouperInstaller.autorun.grouperWhereInstalled");
14274         if (!GrouperInstallerUtils.isBlank(upgradeExistingDirectoryString)) {
14275           grouperInstallDirectoryFile = new File(upgradeExistingDirectoryString);
14276           if (!grouperInstallDirectoryFile.exists() || !grouperInstallDirectoryFile.isDirectory()) {
14277             System.out.println("Error: cant find directory: '" + grouperInstallDirectoryFile.getAbsolutePath() + "'");
14278             continue;
14279           }
14280         } else {
14281           upgradeExistingDirectoryString = defaultDirectory;
14282         }
14283         upgradeExistingDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(upgradeExistingDirectoryString);
14284 
14285         this.upgradeExistingApplicationDirectoryString = upgradeExistingDirectoryString;
14286 
14287         //make sure directory is where the app is
14288         if (!this.appToUpgrade.validateExistingDirectory(this)) {
14289           System.out.println(errorMessage);
14290           continue;
14291         }
14292         //find the resources dir
14293         {
14294           File resourcesDirFile = new File(this.upgradeExistingApplicationDirectoryString + "classes" + File.separator);
14295           if (resourcesDirFile.exists()) {
14296             this.upgradeExistingClassesDirectoryString = resourcesDirFile.getAbsolutePath();
14297           } else {
14298             resourcesDirFile = new File(this.upgradeExistingApplicationDirectoryString + "conf" + File.separator);
14299             if (resourcesDirFile.exists()) {
14300               this.upgradeExistingClassesDirectoryString = resourcesDirFile.getAbsolutePath();
14301             } else {
14302               resourcesDirFile = new File(this.upgradeExistingApplicationDirectoryString + "WEB-INF" + File.separator + "classes" + File.separator);
14303               if (resourcesDirFile.exists()) {
14304                 this.upgradeExistingClassesDirectoryString = resourcesDirFile.getAbsolutePath();
14305               } else {
14306                 this.upgradeExistingClassesDirectoryString = this.upgradeExistingApplicationDirectoryString;
14307               }            
14308             }
14309           }
14310           this.upgradeExistingClassesDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.upgradeExistingClassesDirectoryString);
14311         }
14312         
14313         //find the lib dir
14314         {
14315           File libDirFile = new File(this.upgradeExistingApplicationDirectoryString + "lib" + File.separator + "grouper" + File.separator);
14316           if (libDirFile.exists()) {
14317             this.upgradeExistingLibDirectoryString = libDirFile.getAbsolutePath();
14318           } else {
14319             libDirFile = new File(this.upgradeExistingApplicationDirectoryString + "WEB-INF" + File.separator + "lib" + File.separator);
14320             if (libDirFile.exists()) {
14321               this.upgradeExistingLibDirectoryString = libDirFile.getAbsolutePath();
14322             } else {
14323               libDirFile = new File(this.upgradeExistingApplicationDirectoryString + "lib" + File.separator);
14324               if (libDirFile.exists()) {
14325                 this.upgradeExistingLibDirectoryString = libDirFile.getAbsolutePath();
14326               } else {
14327                 this.upgradeExistingLibDirectoryString = this.upgradeExistingApplicationDirectoryString;
14328               }            
14329             }
14330           }
14331           this.upgradeExistingLibDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.upgradeExistingLibDirectoryString);
14332         }
14333 
14334         //find the bin dir
14335         {
14336           File binDirFile = new File(this.upgradeExistingApplicationDirectoryString + "bin" + File.separator);
14337           if (binDirFile.exists()) {
14338             this.upgradeExistingBinDirectoryString = binDirFile.getAbsolutePath();
14339           } else {
14340             binDirFile = new File(this.upgradeExistingApplicationDirectoryString + "WEB-INF" + File.separator + "bin" + File.separator);
14341             if (binDirFile.exists()) {
14342               this.upgradeExistingBinDirectoryString = binDirFile.getAbsolutePath();
14343             } else {
14344               this.upgradeExistingBinDirectoryString = this.upgradeExistingApplicationDirectoryString;
14345             }
14346           }
14347           this.upgradeExistingBinDirectoryString = GrouperInstallerUtils.fileAddLastSlashIfNotExists(this.upgradeExistingBinDirectoryString);
14348         }
14349 
14350 
14351         return upgradeExistingDirectoryString;
14352       }
14353       
14354       throw new RuntimeException(errorMessage);
14355       
14356     } finally {
14357       //set this back
14358       this.upgradeExistingApplicationDirectoryString = tempUpgradeExistingApplicationDirectoryString;
14359     }
14360   }
14361 
14362   /**
14363    * where classes are in the upgrade directory, ends in file separator
14364    */
14365   private String upgradeExistingClassesDirectoryString;
14366   
14367   /**
14368    * where jars are in the upgrade directory, ends in file separator
14369    */
14370   private String upgradeExistingLibDirectoryString;
14371 
14372   /**
14373    * where bin files (gsh) are in the upgrade directory, ends in file separator
14374    */
14375   private String upgradeExistingBinDirectoryString;
14376   /**
14377    * 
14378    * @param action upgrade or patch
14379    * @return what we are upgrading
14380    */
14381   private AppToUpgrade grouperAppToUpgradeOrPatch(String action) {
14382 
14383     AppToUpgrade appToUpgradeLocal = 
14384         (AppToUpgrade)promptForEnum(
14385             "What do you want to " + action + "?  api, ui, ws, pspng, or psp? ",
14386             "grouperInstaller.autorun.appToUpgrade", AppToUpgrade.class, AppToUpgrade.API, "grouperInstaller.default.appToUpgrade");
14387     return appToUpgradeLocal;
14388   }
14389 
14390   /**
14391    * add something to an xml file
14392    * @param file
14393    * @param addAfterThisRegex 
14394    * @param mustPassTheseRegexes 
14395    * @param newValue 
14396    * @param description of change for sys out print
14397    */
14398   public static void addToXmlFile(File file, String addAfterThisRegex, String[] mustPassTheseRegexes, String newValue, String description) {
14399     if (!file.exists() || file.length() == 0) {
14400       throw new RuntimeException("Why does " + file.getName() + " not exist and have contents? " 
14401           + file.getAbsolutePath());
14402     }
14403     
14404     String fileContents = GrouperInstallerUtils.readFileIntoString(file);
14405     
14406     String newline = GrouperInstallerUtils.newlineFromFile(fileContents);
14407     
14408     Pattern pattern = Pattern.compile(addAfterThisRegex);
14409 
14410     String[] lines = GrouperInstallerUtils.splitLines(fileContents);
14411     
14412     Pattern[] lineMustPassThesePatterns = new Pattern[GrouperInstallerUtils.length(mustPassTheseRegexes)];
14413     
14414     boolean[] hasPassedTheseRegexes = new boolean[lineMustPassThesePatterns.length];
14415     
14416     for (int i=0;i<hasPassedTheseRegexes.length;i++) {
14417       hasPassedTheseRegexes[i] = false;
14418     }
14419     
14420     {
14421       int index = 0;
14422       for (String lineMustHaveRegex : GrouperInstallerUtils.nonNull(mustPassTheseRegexes, String.class)) {
14423         Pattern lineMustHavePattern = Pattern.compile(lineMustHaveRegex);
14424         lineMustPassThesePatterns[index] = lineMustHavePattern;
14425         
14426         index++;
14427       }
14428     }    
14429 
14430     StringBuilder newfile = new StringBuilder();
14431 
14432     boolean madeChange = false;
14433 
14434     OUTER: for (String line : lines) {
14435       line = GrouperInstallerUtils.defaultString(line);
14436       
14437       //lets see if it satisfies all
14438       for (int i=0;i<lineMustPassThesePatterns.length;i++) {
14439         Pattern lineMustHavePattern = lineMustPassThesePatterns[i];
14440         if (lineMustHavePattern.matcher(line).find()) {
14441           hasPassedTheseRegexes[i] = true;
14442         }
14443       }
14444       
14445       //see if we have passed all the prefixes
14446       for (int i=0;i<hasPassedTheseRegexes.length;i++) {
14447         if (!hasPassedTheseRegexes[i]) {
14448           newfile.append(line).append(newline);
14449           continue OUTER;
14450         }
14451       }
14452       
14453       //see if satisfies current, and only add once
14454       Matcher matcher = pattern.matcher(line);
14455       if (!matcher.find() || madeChange) {
14456         newfile.append(line).append(newline);
14457         continue;
14458       }
14459 
14460       //we need to change the value
14461       System.out.println(" - adding " + description + " line: '" + newValue + "'");
14462       newfile.append(line);
14463       newfile.append(newline);
14464       newfile.append(newValue);
14465       newfile.append(newline);
14466       madeChange = true;
14467     }
14468     if (!madeChange) {
14469       throw new RuntimeException("Couldnt find place to add to server.xml!  Are there newlines that werent there before or something?");
14470     }
14471     
14472     GrouperInstallerUtils.writeStringToFile(file, newfile.toString());
14473     
14474   }
14475 
14476   private static String removeElConfigFromPropertiesFile(String fileContents, String propertyName) {
14477     
14478     if (propertyName.endsWith(".elConfig")) {
14479       return fileContents;
14480     }
14481 
14482     String elConfigPropertyName = propertyName + ".elConfig";
14483     
14484     //lets look for property in file
14485     //this is a newline or form feed then some optional whitespace, and the property name
14486     //then some optional whitespace then an equals, then optional whitespace, then some text
14487     String elConfigPropertyNameRegex = GrouperInstallerUtils.replace(elConfigPropertyName, ".", "\\.");
14488     String regex = "[\\n\\r][ \\t]*(" + elConfigPropertyNameRegex + "[ \\t]*=[ \\t]*[^\\n\\r]*)";
14489     Pattern pattern = Pattern.compile(regex, Pattern.DOTALL);
14490     Matcher matcher = pattern.matcher(fileContents);
14491 
14492     if (matcher.find()) {
14493       String previousValue = matcher.group(1);
14494             
14495       int startIndex = matcher.start(1);
14496       
14497       int endIndex = matcher.end(1);
14498       
14499       String newContents = fileContents.substring(0, startIndex);
14500       
14501       //if not the last char
14502       if (endIndex < fileContents.length()-1) {
14503         newContents += fileContents.substring(endIndex, fileContents.length());
14504       }
14505 
14506       //if there is another match, there is a problem
14507       if (matcher.find()) {
14508         throw new RuntimeException("Why are there multiple matches for " + propertyName + " in propertyFile??????");
14509       }
14510 
14511       System.out.println(" - removed property: " 
14512           + elConfigPropertyName);
14513       return newContents;
14514     }
14515     return fileContents;
14516   }
14517   /**
14518    * edit a property in a property file
14519    * @param file
14520    * @param propertyName
14521    * @param propertyValue
14522    * @param createFileIfNotExist 
14523    */
14524   public static void editPropertiesFile(File file, String propertyName, String propertyValue, boolean createFileIfNotExist) {
14525     if (!file.exists()) {
14526       if (createFileIfNotExist) {
14527         System.out.println("Creating file: " + (file == null ? null : file.getAbsolutePath()));
14528         GrouperInstallerUtils.fileCreate(file);
14529       } else {
14530         throw new RuntimeException("Why does " + file.getName() + " not exist and have contents? " 
14531             + file.getAbsolutePath());
14532       }
14533     }
14534     
14535     propertyValue = GrouperInstallerUtils.defaultString(propertyValue);
14536     
14537     String fileContents = GrouperInstallerUtils.readFileIntoString(file);
14538     
14539     String newline = GrouperInstallerUtils.newlineFromFile(fileContents);
14540     
14541     //if it starts with it, add a newline to start so the regexes work
14542     if (!fileContents.startsWith(newline)) {
14543       fileContents = newline + fileContents;
14544     }
14545     
14546     //lets look for property in file
14547     //this is a newline or form feed then some optional whitespace, and the property name
14548     //then some optional whitespace then an equals, then optional whitespace, then some text
14549     String propertyNameRegex = GrouperInstallerUtils.replace(propertyName, ".", "\\.");
14550     String regex = "[\\n\\r][ \\t]*" + propertyNameRegex + "[ \\t]*=[ \\t]*([^\\n\\r]*)";
14551     Pattern pattern = Pattern.compile(regex, Pattern.DOTALL);
14552     Matcher matcher = pattern.matcher(fileContents);
14553 
14554     if (matcher.find()) {
14555       String previousValue = matcher.group(1);
14556       
14557       if (GrouperInstallerUtils.trimToEmpty(previousValue).equals(GrouperInstallerUtils.trim(propertyValue))) {
14558         System.out.println(" - property " + propertyName + " already was set to: " + propertyValue + ", not changing file");
14559         return;
14560       }
14561       
14562       int startIndex = matcher.start(1);
14563       
14564       int endIndex = matcher.end(1);
14565       
14566       String newContents = fileContents.substring(0, startIndex) + propertyValue;
14567       
14568       //if not the last char
14569       if (endIndex < fileContents.length()-1) {
14570         newContents += fileContents.substring(endIndex, fileContents.length());
14571       }
14572 
14573       //if there is another match, there is a problem
14574       if (matcher.find()) {
14575         throw new RuntimeException("Why are there multiple matches for " + propertyName + " in propertyFile: " + file.getAbsolutePath() + "??????");
14576       }
14577 
14578       System.out.println(" - set property: " 
14579           + propertyName + " from: " + previousValue + " to: " + propertyValue);
14580       
14581       newContents = removeElConfigFromPropertiesFile(fileContents, newContents);
14582       
14583       GrouperInstallerUtils.writeStringToFile(file, newContents);
14584       return;
14585     }
14586     
14587     //lets see if it is in a comment
14588     //this is a newline or form feed then some optional whitespace, hash, optional whitespace, and the property name
14589     //then some optional whitespace then an equals, then optional whitespace, then some text
14590     regex = ".*[\\n\\r]([ \\t]*#[ \\t]*)" + propertyNameRegex + "[ \\t]*=[ \\t]*([^\\n\\r]*).*";
14591     pattern = Pattern.compile(regex, Pattern.DOTALL);
14592     matcher = pattern.matcher(fileContents);
14593 
14594     if (matcher.matches()) {
14595       String previousValue = matcher.group(2);
14596       
14597       int startIndexHash = matcher.start(1);
14598       
14599       int endIndexHash = matcher.end(1);
14600 
14601       int startIndex = matcher.start(2);
14602       
14603       int endIndex = matcher.end(2);
14604       
14605       String newContents = fileContents.substring(0, startIndexHash) + fileContents.substring(endIndexHash, startIndex)
14606         + propertyValue;
14607       
14608       //if not the last char
14609       if (endIndex < fileContents.length()-1) {
14610         newContents += fileContents.substring(endIndex, fileContents.length());
14611       }
14612       System.out.println(" - uncommented property: " 
14613           + propertyName + " from: " + previousValue + " to: " + propertyValue);
14614       
14615       newContents = removeElConfigFromPropertiesFile(fileContents, newContents);
14616 
14617       GrouperInstallerUtils.writeStringToFile(file, newContents);
14618       
14619       return;
14620     }
14621     
14622     //it must have not existed
14623     //add a newline..
14624     //add to end in case it was already there, now it will be overwritten
14625     String newContents = fileContents + newline + "# added by grouper-installer" + newline + propertyName + " = " + propertyValue + newline;
14626     
14627     newContents = removeElConfigFromPropertiesFile(newContents, propertyName);
14628 
14629     GrouperInstallerUtils.writeStringToFile(file, newContents);
14630 
14631     System.out.println(" - added to end of property file: " + propertyName + " = " + propertyValue);
14632     
14633   }
14634 
14635   /**
14636    * untar a file to a dir
14637    * @param fileName
14638    * @param autorunPropertiesKeyIfFileExistsUseLocal key in properties file to automatically fill in a value
14639    * @param dirToUntarTo or null to keep in same dir as tarfile
14640    * @return the directory where the files are (assuming has a single dir the same name as the archive)
14641    */
14642   private File untar(final String fileName, final String autorunPropertiesKeyIfFileExistsUseLocal,
14643       File dirToUntarTo) {
14644 
14645     if (!fileName.endsWith(".tar")) {
14646       throw new RuntimeException("File doesnt end in .tar: " + fileName);
14647     }
14648     String untarredFileName = fileName.substring(0, fileName.length() - ".tar".length());
14649     
14650     //ant has -bin which is annoying
14651     if (untarredFileName.endsWith("-bin")) {
14652       untarredFileName = untarredFileName.substring(0, untarredFileName.length() - "-bin".length());
14653     }
14654 
14655     if (dirToUntarTo == null) {
14656       dirToUntarTo = new File(untarredFileName).getParentFile();
14657     }
14658     
14659     File untarredFile = new File(dirToUntarTo.getAbsoluteFile() + File.separator + new File(untarredFileName).getName());
14660     untarredFileName = untarredFile.getAbsolutePath();
14661     
14662     if (untarredFile.exists()) {
14663       
14664       if (this.useAllUntarredDirectories != null && this.useAllUntarredDirectories == true) {
14665         return untarredFile;
14666       }
14667       
14668       System.out.print("Untarred dir exists: " + untarredFileName + ", use untarred dir (t|f)? [t]: ");
14669       boolean useUnzippedFile = readFromStdInBoolean(true, autorunPropertiesKeyIfFileExistsUseLocal);
14670       if (useUnzippedFile) {
14671         
14672         if (this.useAllUntarredDirectories == null) {
14673           System.out.print("Would you like to use all existing untarred directories (t|f)? [t]: ");
14674           this.useAllUntarredDirectories = readFromStdInBoolean(true, "grouperInstaller.autorun.useAllUntarredDirectories");
14675         }
14676         
14677         return untarredFile;
14678       }
14679       
14680       System.out.println("Deleting: " + untarredFileName);
14681       GrouperInstallerUtils.deleteRecursiveDirectory(untarredFileName);
14682     }
14683     
14684     System.out.println("Expanding: " + fileName + " to " + untarredFile.getAbsolutePath());
14685     
14686     final File[] result = new File[1];
14687     
14688     final File DIR_TO_UNTAR_TO = dirToUntarTo;
14689     
14690     Runnable runnable = new Runnable() {
14691 
14692       public void run() {
14693         result[0] = untarHelper(fileName, autorunPropertiesKeyIfFileExistsUseLocal, DIR_TO_UNTAR_TO);
14694       }
14695     };
14696 
14697     GrouperInstallerUtils.threadRunWithStatusDots(runnable, true);
14698 
14699     return result[0];
14700 
14701   }
14702 
14703   /**
14704    * untar a file to a dir
14705    * @param fileName
14706    * @param autorunPropertiesKeyIfFileExistsUseLocal key in properties file to automatically fill in a value
14707    * @param dirToUntarTo or null to keep in same dir as tarfile
14708    * @return the directory where the files are (assuming has a single dir the same name as the archive)
14709    */
14710   @SuppressWarnings("resource")
14711   private static File untarHelper(String fileName, String autorunPropertiesKeyIfFileExistsUseLocal, File dirToUntarTo) {
14712     TarArchiveInputStream tarArchiveInputStream = null;
14713     
14714     String untarredFileName = fileName.substring(0, fileName.length() - ".tar".length());
14715     
14716     //ant has -bin which is annoying
14717     if (untarredFileName.endsWith("-bin")) {
14718       untarredFileName = untarredFileName.substring(0, untarredFileName.length() - "-bin".length());
14719     }
14720     
14721     if (dirToUntarTo == null) {
14722       dirToUntarTo = new File(untarredFileName).getParentFile();
14723     }
14724 
14725     File untarredFile = new File(dirToUntarTo.getAbsoluteFile() + File.separator + new File(untarredFileName).getName());
14726 
14727     try {
14728 
14729       tarArchiveInputStream = new TarArchiveInputStream(new FileInputStream(fileName));
14730       
14731       while (true) {
14732   
14733         TarArchiveEntry tarArchiveEntry = tarArchiveInputStream.getNextTarEntry();
14734         if (tarArchiveEntry == null) {
14735           break;
14736         }
14737         
14738         //System.out.println("Entry: " + tarArchiveEntry.getName()
14739         //    + ", isDirectory: " + tarArchiveEntry.isDirectory()
14740         //    + ", isFile: " + tarArchiveEntry.isFile());
14741         String fileEntryName = dirToUntarTo.getAbsolutePath() + File.separator + tarArchiveEntry.getName();
14742         File tarEntryFile = new File(fileEntryName);
14743         
14744         if (tarArchiveEntry.isDirectory()) {
14745           if (!tarEntryFile.exists() && !tarEntryFile.mkdirs()) {
14746             throw new RuntimeException("Cant create dirs: " + tarEntryFile.getAbsolutePath());
14747           }
14748           continue;
14749         }
14750         
14751         byte[] content = new byte[(int)tarArchiveEntry.getSize()];
14752 
14753         int size = tarArchiveInputStream.read(content, 0, content.length);
14754         
14755         //for some reason we get an error when 0 bytes...
14756         if (size != content.length && (!(size == -1 && content.length == 0))) {
14757           throw new RuntimeException("Didnt read the right amount of bytes: " + size 
14758               + ", should have been: " + content.length + " on entry: " + tarArchiveEntry.getName());
14759         }
14760         
14761         ByteArrayInputStream byteArrayInputStream = null;
14762         FileOutputStream fileOutputStream = null;
14763         
14764         try {
14765           
14766           //create parent directories
14767           if (!tarEntryFile.getParentFile().exists() && !tarEntryFile.getParentFile().mkdirs()) {
14768             throw new RuntimeException("Cant create dirs: " + tarEntryFile.getParentFile().getAbsolutePath());
14769           }
14770 
14771           fileOutputStream = new FileOutputStream(tarEntryFile);
14772           byteArrayInputStream = new ByteArrayInputStream(content);
14773           GrouperInstallerUtils.copy(byteArrayInputStream, fileOutputStream);
14774           
14775         } catch (Exception e) {
14776           throw new RuntimeException("Problem with entry: " + tarArchiveEntry.getName(), e);
14777         } finally {
14778           GrouperInstallerUtils.closeQuietly(byteArrayInputStream);
14779           GrouperInstallerUtils.closeQuietly(fileOutputStream);
14780         }
14781         
14782       }
14783     } catch (Exception e) {
14784       throw new RuntimeException("Error untarring: " + fileName, e);
14785     } finally {
14786       GrouperInstallerUtils.closeQuietly(tarArchiveInputStream);
14787     }
14788     return untarredFile;
14789   }
14790 
14791   /**
14792    * unzip a file, this is for .zip files
14793    * @param fileName
14794    * @param autorunPropertiesKeyIfFileExistsUseLocal key in properties file to automatically fill in a value
14795    * @return the unzipped file
14796    */
14797   private static File unzipFromZip(final String fileName, final String autorunPropertiesKeyIfFileExistsUseLocal) {
14798 
14799     if (!fileName.endsWith(".zip")) {
14800       throw new RuntimeException("File doesnt end in .zip: " + fileName);
14801     }
14802     String unzippedFileName = fileName.substring(0, fileName.length() - ".zip".length());
14803 
14804     File unzippedDir = new File(unzippedFileName);
14805 
14806     if (unzippedDir.exists()) {
14807       System.out.print("Unzipped dir exists: " + unzippedFileName + ", use unzipped dir (t|f)? [t]: ");
14808       boolean useUnzippedFile = readFromStdInBoolean(true, autorunPropertiesKeyIfFileExistsUseLocal);
14809       if (useUnzippedFile) {
14810         return unzippedDir;
14811       }
14812       System.out.println("Deleting: " + unzippedFileName);
14813       GrouperInstallerUtils.deleteRecursiveDirectory(unzippedFileName);
14814     } else {
14815       if (!unzippedDir.mkdir()) {
14816         throw new RuntimeException("Cant make dir: " + unzippedDir.getAbsolutePath());
14817       }
14818     }
14819 
14820     System.out.println("Unzipping: " + fileName);
14821 
14822     final File[] result = new File[1];
14823     
14824     Runnable runnable = new Runnable() {
14825 
14826       public void run() {
14827         result[0] = unzipFromZipHelper(fileName, autorunPropertiesKeyIfFileExistsUseLocal);
14828       }
14829     };
14830 
14831     GrouperInstallerUtils.threadRunWithStatusDots(runnable, true);
14832 
14833     return result[0];
14834 
14835   }
14836 
14837   /**
14838    * unzip a file, this is for .zip files
14839    * @param fileName
14840    * @param autorunPropertiesKeyIfFileExistsUseLocal key in properties file to automatically fill in a value
14841    * @return the unzipped file
14842    */
14843   private static File unzipFromZipHelper(String fileName, String autorunPropertiesKeyIfFileExistsUseLocal) {
14844 
14845     String unzippedFileName = fileName.substring(0, fileName.length() - ".zip".length());
14846 
14847     File unzippedDir = new File(unzippedFileName);
14848 
14849     ZipFile zipFile = null;
14850     try {
14851       zipFile = new ZipFile(fileName);
14852       Enumeration<? extends ZipEntry> entries = zipFile.entries();
14853       while (entries.hasMoreElements()) {
14854         ZipEntry entry = entries.nextElement();
14855         File entryDestination = new File(unzippedDir, entry.getName());
14856         if (entry.isDirectory()) {
14857           entryDestination.mkdirs();
14858         } else {
14859           entryDestination.getParentFile().mkdirs();
14860           InputStream in = zipFile.getInputStream(entry);
14861           OutputStream out = new FileOutputStream(entryDestination);
14862           try {
14863             IOUtils.copy(in, out);
14864           } finally {
14865             GrouperInstallerUtils.closeQuietly(in);
14866             GrouperInstallerUtils.closeQuietly(out);
14867           }
14868         }
14869       }
14870     } catch (IOException ioe) {
14871       throw new RuntimeException(ioe);
14872     } finally {
14873       GrouperInstallerUtils.closeQuietly(zipFile);
14874     }
14875 
14876     return unzippedDir;
14877 
14878   }
14879 
14880   /**
14881    * unzip a file to another file, this is for .gz files
14882    * @param fileName
14883    * @param autorunPropertiesKeyIfFileExistsUseLocal key in properties file to automatically fill in a value
14884    * @return the unzipped file
14885    */
14886   private File unzip(final String fileName, final String autorunPropertiesKeyIfFileExistsUseLocal) {
14887 
14888     if (!fileName.endsWith(".gz")) {
14889       throw new RuntimeException("File doesnt end in .gz: " + fileName);
14890     }
14891     String unzippedFileName = fileName.substring(0, fileName.length() - ".gz".length());
14892     
14893     File unzippedFile = new File(unzippedFileName);
14894     if (unzippedFile.exists()) {
14895       
14896       if (this.useAllUnzippedFiles != null && this.useAllUnzippedFiles == true) {
14897         return unzippedFile;
14898       }
14899       
14900       System.out.print("Unzipped file exists: " + unzippedFileName + ", use unzipped file (t|f)? [t]: ");
14901       boolean useUnzippedFile = readFromStdInBoolean(true, autorunPropertiesKeyIfFileExistsUseLocal);
14902       if (useUnzippedFile) {
14903         if (this.useAllUnzippedFiles == null) {
14904           System.out.print("Would you like to use all existing unzipped files (t|f)? [t]: ");
14905           this.useAllUnzippedFiles = readFromStdInBoolean(true, "grouperInstaller.autorun.useAllUnzippedFiles");
14906         }
14907         
14908         return unzippedFile;
14909       }
14910       System.out.println("Deleting: " + unzippedFileName);
14911       if (!unzippedFile.delete()) {
14912         throw new RuntimeException("Cant delete file: " + unzippedFileName);
14913       }
14914     }
14915 
14916     System.out.println("Unzipping: " + fileName);
14917     
14918     final File[] result = new File[1];
14919     
14920     Runnable runnable = new Runnable() {
14921 
14922       public void run() {
14923         result[0] = unzipHelper(fileName, autorunPropertiesKeyIfFileExistsUseLocal);
14924       }
14925     };
14926 
14927     GrouperInstallerUtils.threadRunWithStatusDots(runnable, true);
14928 
14929     return result[0];
14930 
14931   }
14932 
14933   /**
14934    * unzip a file to another file, this is for .gz files
14935    * @param fileName
14936    * @param autorunPropertiesKeyIfFileExistsUseLocal key in properties file to automatically fill in a value
14937    * @return the unzipped file
14938    */
14939   private static File unzipHelper(String fileName, String autorunPropertiesKeyIfFileExistsUseLocal) {
14940 
14941     String unzippedFileName = fileName.substring(0, fileName.length() - ".gz".length());
14942     File unzippedFile = new File(unzippedFileName);
14943 
14944     GzipCompressorInputStream gzipCompressorInputStream = null;
14945     FileOutputStream fileOutputStream = null;
14946     try {
14947       gzipCompressorInputStream = new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(new File(fileName))));
14948       fileOutputStream = new FileOutputStream(unzippedFile);
14949       GrouperInstallerUtils.copy(gzipCompressorInputStream, fileOutputStream);
14950     } catch (Exception e) {
14951       throw new RuntimeException("Cant unzip file: " + fileName, e);
14952     } finally {
14953       GrouperInstallerUtils.closeQuietly(gzipCompressorInputStream);
14954       GrouperInstallerUtils.closeQuietly(fileOutputStream);
14955     }
14956     return unzippedFile;
14957   }
14958 
14959   /**
14960    * 
14961    * @return the file of the directory of the psp
14962    */
14963   private File downloadPsp() {
14964     String urlToDownload = GrouperInstallerUtils.propertiesValue("download.server.url", true);
14965     
14966     if (!urlToDownload.endsWith("/")) {
14967       urlToDownload += "/";
14968     }
14969     urlToDownload += "release/";
14970 
14971     String pspFileName = "grouper.psp-" + this.version + ".tar.gz";
14972     urlToDownload += this.version + "/" + pspFileName;
14973 
14974     File pspFile = new File(this.grouperTarballDirectoryString + pspFileName);
14975     
14976     downloadFile(urlToDownload, pspFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalPspDownloadTarEtc");
14977 
14978     return pspFile;
14979   }
14980   
14981   /**
14982    * 
14983    * @return the file of the directory of the psp
14984    */
14985   private File downloadPspng() {
14986     String urlToDownload = GrouperInstallerUtils.propertiesValue("download.server.url", true);
14987     
14988     if (!urlToDownload.endsWith("/")) {
14989       urlToDownload += "/";
14990     }
14991     urlToDownload += "release/";
14992 
14993     String pspFileName = "grouper.pspng-" + this.version + ".tar.gz";
14994     urlToDownload += this.version + "/" + pspFileName;
14995 
14996     File pspFile = new File(this.grouperTarballDirectoryString + pspFileName);
14997     
14998     downloadFile(urlToDownload, pspFile.getAbsolutePath(), "grouperInstaller.autorun.useLocalPspngDownloadTarEtc");
14999 
15000     return pspFile;
15001   }
15002   
15003   /**
15004    * upgrade the ws
15005    */
15006   private void upgradeWs() {
15007   
15008     this.upgradeApiPreRevertPatch();
15009 
15010     System.out.println("You need to revert all patches to upgrade");
15011     this.patchRevertWs();
15012     
15013     System.out.println("\n##################################");
15014     System.out.println("Upgrading WS\n");
15015     
15016     //copy the jars there
15017     System.out.println("\n##################################");
15018     System.out.println("Upgrading WS jars\n");
15019   
15020     this.upgradeJars(new File(this.untarredWsDir + File.separator + 
15021         "grouper-ws" + File.separator + "build" + File.separator + "dist" + File.separator + "grouper-ws"
15022         + File.separator + "WEB-INF" + File.separator + "lib" + File.separator));
15023   
15024     System.out.println("\n##################################");
15025     System.out.println("Upgrading WS files\n");
15026   
15027     //copy files there
15028     this.copyFiles(this.untarredWsDir + File.separator + 
15029         "grouper-ws" + File.separator + "build" + File.separator + "dist" + File.separator + "grouper-ws"
15030         + File.separator,
15031         this.upgradeExistingApplicationDirectoryString,
15032         GrouperInstallerUtils.toSet("WEB-INF/lib", "WEB-INF/web.xml", "WEB-INF/web.wsTomcatAuthn.xml", "WEB-INF/server.wsTomcatAuthn.xml", "WEB-INF/classes",
15033             "WEB-INF/bin/gsh", "WEB-INF/bin/gsh.bat", "WEB-INF/bin/gsh.sh"));
15034 
15035     {
15036       boolean hadChange = false;
15037       for (String gshName : new String[]{"gsh", "gsh.bat", "gsh.sh"}) {
15038         File newGshFile = new File(this.untarredWsDir + File.separator + "grouper-ws" + File.separator 
15039             + "build" + File.separator + "dist" + File.separator + "grouper-ws"
15040             + File.separator + "WEB-INF" + File.separator + "bin" 
15041             + File.separator + gshName);
15042   
15043         File existingGshFile = new File(this.upgradeExistingApplicationDirectoryString 
15044             + File.separator + "WEB-INF" + File.separator + "bin" + File.separator + gshName);
15045   
15046         if (!GrouperInstallerUtils.contentEquals(newGshFile, existingGshFile)) {
15047           this.backupAndCopyFile(newGshFile, existingGshFile, true);
15048           if (!GrouperInstallerUtils.equals("gsh.bat", gshName)) {
15049             hadChange = true;
15050           }
15051         }
15052         
15053       }
15054       if (hadChange) {
15055         //set executable and dos2unix
15056         gshExcutableAndDos2Unix(this.upgradeExistingApplicationDirectoryString + "WEB-INF" 
15057             + File.separator + "bin" 
15058             + File.separator);
15059       }
15060     }
15061     
15062     upgradeWebXml(new File(this.untarredWsDir + File.separator + "grouper-ws" 
15063         + File.separator + "build" + File.separator + "dist" + File.separator + "grouper-ws"
15064         + File.separator + "WEB-INF" + File.separator + "web.xml"),
15065             new File(this.upgradeExistingApplicationDirectoryString 
15066                 + File.separator + "WEB-INF" + File.separator + "web.xml"));
15067     
15068     System.out.println("\n##################################");
15069     System.out.println("Upgrading WS config files\n");
15070 
15071     this.compareUpgradePropertiesFile(this.grouperWsBasePropertiesFile, 
15072         new File(this.untarredWsDir + File.separator + 
15073             "grouper-ws" + File.separator + "build" + File.separator + "dist" + File.separator + "grouper-ws"
15074             + File.separator + "WEB-INF" + File.separator + "classes" + File.separator + "grouper-ws.base.properties"),
15075         this.grouperWsPropertiesFile,
15076         this.grouperWsExamplePropertiesFile, null, "grouperInstaller.autorun.removeRedundantPropetiesFromGrouperWsProperties"
15077       );
15078 
15079     this.upgradeApiPostRevertPatch();
15080     
15081     //patch it
15082     this.patchWs();
15083 
15084   }
15085   /**
15086    * get the patches available to apply that are not already applied
15087    * @param thisAppToUpgrade app to upgrade to check
15088    * @return true if up to date, false if not
15089    */
15090   private boolean patchStatus(AppToUpgrade thisAppToUpgrade) {
15091   
15092     if (thisAppToUpgrade == AppToUpgrade.CLIENT) {
15093       throw new RuntimeException("Cant get status on " + thisAppToUpgrade);
15094     }
15095     
15096     Properties patchesExistingProperties = patchExistingProperties();
15097     
15098     String grouperVersion = this.grouperVersionOfJar().toString();
15099     
15100     grouperVersion = GrouperInstallerUtils.replace(grouperVersion, ".", "_");
15101     
15102     boolean foundNewPatch = false;
15103     
15104     OUTER: for (int i=0;i<1000;i++) {
15105       
15106       //grouper_v2_2_1_api_patch_0.state
15107       String keyBase = "grouper_v" + grouperVersion + "_" + thisAppToUpgrade.name().toLowerCase() + "_patch_" + i;
15108       System.out.println("\n################ Checking patch " + keyBase);
15109       String key = keyBase + ".state";
15110   
15111       String value = patchesExistingProperties.getProperty(key);
15112 
15113       if (!GrouperInstallerUtils.isBlank(value)) {
15114 
15115         GrouperInstallerPatchStatus grouperInstallerPatchStatus = GrouperInstallerPatchStatus.valueOfIgnoreCase(value, true, true);
15116 
15117         switch (grouperInstallerPatchStatus) {
15118           case applied:
15119             
15120             System.out.println("Patch: " + keyBase + ": was applied on: " + patchesExistingProperties.getProperty(keyBase + ".date"));
15121             break;
15122             
15123           case skippedPermanently:
15124             
15125             foundNewPatch = true;
15126             System.out.println("Patch: " + keyBase + ": was skipped permanently on: " + patchesExistingProperties.getProperty(keyBase + ".date"));
15127             break;
15128             
15129           case skippedTemporarily:
15130   
15131             foundNewPatch = true;
15132             System.out.println("Patch: " + keyBase + ": was skipped termporarily on: " + patchesExistingProperties.getProperty(keyBase + ".date"));
15133             break;
15134   
15135           case reverted:
15136   
15137             foundNewPatch = true;
15138             System.out.println("Patch: " + keyBase + ": was reverted on: " + patchesExistingProperties.getProperty(keyBase + ".date"));
15139             break;
15140   
15141           case error:
15142   
15143             foundNewPatch = true;
15144             System.out.println("Patch: " + keyBase + ": had an error installing on: " + patchesExistingProperties.getProperty(keyBase + ".date"));
15145             break;
15146   
15147           default:
15148             throw new RuntimeException("Not expecting: " + grouperInstallerPatchStatus);
15149         }
15150         
15151       }
15152   
15153       //lets see if it exists on the server
15154       File patchUntarredDir = downloadAndUnzipPatch(keyBase);
15155       
15156       //if no more patches
15157       if (patchUntarredDir == null) {
15158         System.out.println("");
15159         break OUTER;
15160       }
15161       
15162       //lets get the description:
15163       //  # will show up on screen so user knows what it is
15164       //  description = This patch fixes GRP-1080: browse folders refresh button only works in chrome, not other browsers
15165       //
15166       //  # patches that this patch is dependant on (comma separated)
15167       //  dependencies = 
15168       //
15169       //  # low, medium, or high risk to applying the patch
15170       //  risk = low
15171       //
15172       //  # is this is a security patch (true or false)
15173       //  security = false
15174       //
15175       //  # if this patch requires restart of processes (true or false)
15176       //  requiresRestart = false
15177       Properties patchProperties = GrouperInstallerUtils.propertiesFromFile(new File(patchUntarredDir.getAbsoluteFile() + File.separator + keyBase + ".properties"));
15178   
15179       foundNewPatch = true;
15180   
15181       // check dependencies
15182       {
15183         String[] dependencies = GrouperInstallerUtils.splitTrim(patchProperties.getProperty("dependencies"), ",");
15184   
15185         for (String dependency : GrouperInstallerUtils.nonNull(dependencies, String.class)) {
15186           if (!this.patchesInstalled.contains(dependency)) {
15187             System.out.println("Cannot install patch " + keyBase + " since it is dependent on a patch which is not installed: " + dependency);
15188           }
15189         }
15190       }
15191       
15192       boolean securityRelated = GrouperInstallerUtils.booleanValue(patchProperties.getProperty("security"), false);
15193       boolean requiresRestart = GrouperInstallerUtils.booleanValue(patchProperties.getProperty("requiresRestart"), true);
15194       
15195       //print description
15196       System.out.println("Patch " + keyBase + " is " + patchProperties.getProperty("risk") + " risk, "
15197           + (securityRelated ? "is a security patch" : "is not a security patch"));
15198       System.out.println("Patch " + keyBase + (requiresRestart ? " requires" : " does not require") + " a restart");
15199       System.out.println(patchProperties.getProperty("description") + "\n");      
15200     }
15201   
15202     if (!foundNewPatch) {
15203       System.out.println("There are no new " + thisAppToUpgrade + " patches to install");
15204       return true;
15205     }
15206     
15207     return false;
15208   }
15209   /**
15210    * build PSPNG
15211    * @param pspngDir
15212    */
15213   private void buildPspng(File pspngDir) {
15214 
15215     if (!pspngDir.exists() || pspngDir.isFile()) {
15216       throw new RuntimeException("Cant find psp: " + pspngDir.getAbsolutePath());
15217     }
15218     
15219     File pspngBuildToDir = new File(pspngDir.getAbsolutePath() 
15220         + File.separator + "target" + File.separator + "classes");
15221     
15222     boolean rebuildPspng = true;
15223     
15224     if (pspngBuildToDir.exists()) {
15225       System.out.print("The PSPNG has been built in the past, do you want it rebuilt? (t|f) [t]: ");
15226       rebuildPspng = readFromStdInBoolean(true, "grouperInstaller.autorun.rebuildPspngAfterHavingBeenBuilt");
15227     }
15228     
15229     if (!rebuildPspng) {
15230       return;
15231     }
15232     
15233     List<String> commands = new ArrayList<String>();
15234     
15235 //    \bin\mvn compile -DskipTests
15236     addMavenCommands(commands);
15237 
15238     //put 'compile -DskipTests' in there so it wont run tests which we dont want to do
15239     // dependency:copy-dependencies package -DskipTests
15240     //not compile
15241     commands.add("dependency:copy-dependencies");
15242     commands.add("package");
15243     commands.add("-DskipTests");
15244     commands.add("-Drat.ignoreErrors=true");
15245     commands.add("-Dlicense.skip=true");
15246         
15247     System.out.println("\n##################################");
15248     System.out.println("Building PSPNG with command:\n" + pspngDir.getAbsolutePath() + "> " 
15249         + convertCommandsIntoCommand(commands) + "\n");
15250 
15251     CommandResult commandResult = GrouperInstallerUtils.execCommand(GrouperInstallerUtils.toArray(commands, String.class),
15252         true, true, null, new File(pspngDir.getAbsolutePath()), null, true);
15253     
15254     if (!GrouperInstallerUtils.isBlank(commandResult.getErrorText())) {
15255       System.out.println("stderr: " + commandResult.getErrorText());
15256     }
15257     if (!GrouperInstallerUtils.isBlank(commandResult.getOutputText())) {
15258       System.out.println("stdout: " + commandResult.getOutputText());
15259     }
15260 
15261     System.out.println("\nEnd building PSPNG");
15262     System.out.println("##################################\n");
15263     
15264   }
15265 
15266   /**
15267    * 
15268    */
15269   private void upgradeSourcesXmlToProperties() {
15270   
15271     //dont do this if less than 2.3.1
15272     if (new GiGrouperVersionersion(this.version).lessThanArg(new GiGrouperVersion("2.3.1"))) {
15273       return;
15274     }
15275     
15276     //this file is done
15277     File sourcesXmlFile = new File(this.grouperPropertiesFile.getParentFile().getAbsolutePath() + File.separator + "sources.xml");
15278 
15279     if (!sourcesXmlFile.exists()) {
15280       return;
15281     }
15282       
15283     
15284     System.out.print("Do you want to convert from sources.xml to subject.properties, note you need to do this to upgrade (t|f)? [t]: ");
15285     boolean convert = readFromStdInBoolean(true, "grouperInstaller.autorun.convertSourcesXmlToProperties");
15286   
15287     if (!convert) {
15288       System.out.println("Note: grouper will not run, but whatever you want to do!!!!");
15289     }
15290     File bakFile = null;
15291     if (this.subjectPropertiesFile.exists()) {
15292       //see if there is anything in it
15293       Properties grouperCacheProperties = GrouperInstallerUtils.propertiesFromFile(this.subjectPropertiesFile);
15294       if (grouperCacheProperties.size() > 0) {
15295         bakFile = this.backupAndDeleteFile(this.grouperCachePropertiesFile, true);
15296       }
15297     }
15298     
15299     URL sourcesXmlUrl = null;
15300     
15301     try {
15302       sourcesXmlUrl = sourcesXmlFile.toURI().toURL();
15303     } catch (Exception e) {
15304       throw new RuntimeException("Problem with sources.xml: " + (sourcesXmlFile == null ? null : sourcesXmlFile.getAbsoluteFile()), e);
15305     }
15306     
15307     //convert
15308     convertSourcesXmlToProperties(this.subjectPropertiesFile, sourcesXmlUrl);
15309     
15310     File subjectBakFile = bakFile(this.subjectPropertiesFile);
15311     GrouperInstallerUtils.copyFile(this.subjectPropertiesFile, subjectBakFile, true);
15312     this.backupAndDeleteFile(sourcesXmlFile, true);
15313     
15314     {
15315       File sourcesExampleXmlFile = new File(this.grouperPropertiesFile.getParentFile().getAbsolutePath() + File.separator + "sources.example.xml");
15316       if (sourcesExampleXmlFile.exists()) {
15317         this.backupAndDeleteFile(sourcesExampleXmlFile, true);
15318       }
15319     }
15320     
15321     if (bakFile != null) {
15322       System.out.println("Note, you had settings in your subject.properties (not common), this file has been moved to: " + bakFile.getAbsolutePath());
15323       System.out.println("Merge your settings from that file to " + this.subjectPropertiesFile.getAbsolutePath());
15324       System.out.print("Press <enter> to continue: ");
15325       readFromStdIn("grouperInstaller.autorun.convertSourcesXmlToPropertiesHadPropertiesInFile");
15326     }
15327   }
15328 
15329   /**
15330    * 
15331    * @param grouperCacheBasePropertiesFile 
15332    * @param grouperCachePropertiesFile
15333    * @param ehcacheXmlUrl
15334    */
15335   public static void convertEhcacheXmlToProperties(File grouperCacheBasePropertiesFile, File grouperCachePropertiesFile, URL ehcacheXmlUrl) {
15336   
15337     //look at base properties
15338     Properties grouperCacheProperties = grouperCachePropertiesFile.exists() ? 
15339         GrouperInstallerUtils.propertiesFromFile(grouperCachePropertiesFile) : new Properties();
15340   
15341     if (!grouperCacheBasePropertiesFile.exists()) {
15342       throw new RuntimeException(grouperCacheBasePropertiesFile.getAbsolutePath() + " must exist and does not!");
15343     }
15344     
15345     if (grouperCacheProperties.size() > 0) {
15346       throw new RuntimeException(grouperCachePropertiesFile.getAbsolutePath() + " exists and must not.  Delete the file and run this again!");
15347     }
15348   
15349     if (!grouperCachePropertiesFile.getParentFile().exists() || !grouperCachePropertiesFile.getParentFile().isDirectory()) {
15350       throw new RuntimeException(grouperCachePropertiesFile.getParentFile().getAbsolutePath() + " must exist and must be a directory");
15351     }
15352     
15353     //look at base properties
15354     Properties grouperCacheBaseProperties = GrouperInstallerUtils.propertiesFromFile(grouperCacheBasePropertiesFile);
15355     
15356     StringBuilder grouperEhcachePropertiesContents = new StringBuilder();
15357     
15358     grouperEhcachePropertiesContents.append(
15359               "# Copyright 2016 Internet2\n"
15360             + "#\n"
15361             + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n"
15362             + "# you may not use this file except in compliance with the License.\n"
15363             + "# You may obtain a copy of the License at\n"
15364             + "#\n"
15365             + "#   http://www.apache.org/licenses/LICENSE-2.0\n"
15366             + "#\n"
15367             + "# Unless required by applicable law or agreed to in writing, software\n"
15368             + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n"
15369             + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n"
15370             + "# See the License for the specific language governing permissions and\n"
15371             + "# limitations under the License.\n"
15372             + "\n"
15373             + "#\n"
15374             + "# Grouper Cache Configuration\n"
15375             + "#\n"
15376             + "\n"
15377             + "# The grouper cache config uses Grouper Configuration Overlays (documented on wiki)\n"
15378             + "# By default the configuration is read from grouper.cache.base.properties\n"
15379             + "# (which should not be edited), and the grouper.cache.properties overlays\n"
15380             + "# the base settings.  See the grouper.cache.base.properties for the possible\n"
15381             + "# settings that can be applied to the grouper.cache.properties\n\n"
15382         );
15383   
15384     {
15385       // <diskStore path="java.io.tmpdir"/>
15386       NodeList diskStoreNodeList = GrouperInstallerUtils.xpathEvaluate(ehcacheXmlUrl, "/ehcache/diskStore");
15387       if (diskStoreNodeList.getLength() != 1) {
15388         throw new RuntimeException("Expecting one diskStore element");
15389       }
15390   
15391       Element element = (Element)diskStoreNodeList.item(0);
15392   
15393       NamedNodeMap configuredNamedNodeMap = element.getAttributes();
15394       if (configuredNamedNodeMap.getLength() != 1 || !"path".equals(configuredNamedNodeMap.item(0).getNodeName())) {
15395         throw new RuntimeException("Expecting one diskStore attribute: path");
15396       }
15397       
15398       String path = element.getAttribute("path");
15399       
15400       if (!"java.io.tmpdir".equals(path)) {
15401         grouperEhcachePropertiesContents.append("grouper.cache.diskStorePath = " + path + "\n\n");
15402       }
15403       
15404     }    
15405   
15406     {
15407       //  <defaultCache
15408       //    maxElementsInMemory="1000"
15409       //    eternal="false"
15410       //    timeToIdleSeconds="10"
15411       //    timeToLiveSeconds="10"
15412       //    overflowToDisk="false"
15413       //    statistics="false"
15414       //  />
15415       
15416       NodeList diskStoreNodeList = GrouperInstallerUtils.xpathEvaluate(ehcacheXmlUrl, "/ehcache/defaultCache");
15417       if (diskStoreNodeList.getLength() != 1) {
15418         throw new RuntimeException("Expecting one defaultCache element");
15419       }
15420   
15421       Element element = (Element)diskStoreNodeList.item(0);
15422   
15423       NamedNodeMap configuredNamedNodeMap = element.getAttributes();
15424       
15425       if (configuredNamedNodeMap.getLength() != 6) {
15426         throw new RuntimeException("Expecting defaultCache with these attributes: maxElementsInMemory, "
15427             + "eternal, timeToIdleSeconds, timeToLiveSeconds, overflowToDisk, statistics");
15428       }
15429   
15430       boolean madeChanges = false;
15431       
15432       for (int i=0;i<configuredNamedNodeMap.getLength(); i++) {
15433         
15434         String attributeName = configuredNamedNodeMap.item(i).getNodeName();
15435         String value = element.getAttribute(attributeName);
15436   
15437         if ("maxElementsInMemory".equals(attributeName)) {
15438           if (!"1000".equals(value)) {
15439             grouperEhcachePropertiesContents.append("cache.defaultCache.maxElementsInMemory = " + value + "\n");
15440             madeChanges = true;
15441           }
15442         } else if ("eternal".equals(attributeName)) {
15443           if (!"false".equals(value)) {
15444             grouperEhcachePropertiesContents.append("cache.defaultCache.eternal = " + value + "\n");
15445             madeChanges = true;
15446           }
15447         } else if ("timeToIdleSeconds".equals(attributeName)) {
15448           if (!"10".equals(value)) {
15449             grouperEhcachePropertiesContents.append("cache.defaultCache.timeToIdleSeconds = " + value + "\n");
15450             madeChanges = true;
15451           }
15452           
15453         } else if ("timeToLiveSeconds".equals(attributeName)) {
15454           if (!"10".equals(value)) {
15455             grouperEhcachePropertiesContents.append("cache.defaultCache.timeToLiveSeconds = " + value + "\n");
15456             madeChanges = true;
15457           }
15458           
15459         } else if ("overflowToDisk".equals(attributeName)) {
15460           if (!"false".equals(value)) {
15461             grouperEhcachePropertiesContents.append("cache.defaultCache.overflowToDisk = " + value + "\n");
15462             madeChanges = true;
15463           }
15464           
15465         } else if ("statistics".equals(attributeName)) {
15466           if (!"false".equals(value)) {
15467             grouperEhcachePropertiesContents.append("cache.defaultCache.statistics = " + value + "\n");
15468             madeChanges = true;
15469           }
15470           
15471         } else {
15472           throw new RuntimeException("Not expecting attribuet defaultCache " + attributeName);
15473         }
15474       }
15475   
15476       if (madeChanges) {
15477         grouperEhcachePropertiesContents.append("\n");
15478       }
15479       
15480     }
15481     
15482     NodeList nodeList = GrouperInstallerUtils.xpathEvaluate(ehcacheXmlUrl, "/ehcache/cache");
15483     
15484     Set<String> usedKeys = new HashSet<String>();
15485     
15486     for (int i=0;i<nodeList.getLength();i++) {
15487       
15488       Element element = (Element)nodeList.item(i);
15489   
15490       //  <cache  name="edu.internet2.middleware.grouper.internal.dao.hib3.Hib3MemberDAO.FindBySubject"
15491       //      maxElementsInMemory="5000"
15492       //      eternal="false"
15493       //      timeToIdleSeconds="5"
15494       //      timeToLiveSeconds="10"
15495       //      overflowToDisk="false"  
15496       //      statistics="false"
15497       //  />
15498       
15499       String name = element.getAttribute("name");
15500       Integer maxElementsInMemory = GrouperInstallerUtils.intObjectValue(element.getAttribute("maxElementsInMemory"), true);
15501       Boolean eternal = GrouperInstallerUtils.booleanObjectValue(element.getAttribute("eternal"));
15502       Integer timeToIdleSeconds = GrouperInstallerUtils.intObjectValue(element.getAttribute("timeToIdleSeconds"), true);
15503       Integer timeToLiveSeconds = GrouperInstallerUtils.intObjectValue(element.getAttribute("timeToLiveSeconds"), true);
15504       Boolean overflowToDisk = GrouperInstallerUtils.booleanObjectValue(element.getAttribute("overflowToDisk"));
15505       Boolean statistics = GrouperInstallerUtils.booleanObjectValue(element.getAttribute("statistics"));
15506   
15507       //any attributes we dont expect?
15508       NamedNodeMap configuredNamedNodeMap = element.getAttributes();
15509       //see which attributes are new or changed
15510       for (int j=0;j<configuredNamedNodeMap.getLength();j++) {
15511         Node configuredAttribute = configuredNamedNodeMap.item(j);
15512         if (!configuredAttribute.getNodeName().equals("name")
15513             && !configuredAttribute.getNodeName().equals("maxElementsInMemory")
15514             && !configuredAttribute.getNodeName().equals("eternal")
15515             && !configuredAttribute.getNodeName().equals("timeToIdleSeconds")
15516             && !configuredAttribute.getNodeName().equals("timeToLiveSeconds")
15517             && !configuredAttribute.getNodeName().equals("overflowToDisk")
15518             && !configuredAttribute.getNodeName().equals("statistics")) {
15519           throw new RuntimeException("Cant process attribute: '" + configuredAttribute.getNodeName() + "'");
15520         }
15521       }
15522       
15523       String key = convertEhcacheNameToPropertiesKey(name, usedKeys);
15524       
15525       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.name = edu.internet2.middleware.grouper.internal.dao.hib3.Hib3GroupDAO
15526       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.maxElementsInMemory = 500
15527       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.eternal = false
15528       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.timeToIdleSeconds = 1
15529       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.timeToLiveSeconds = 1
15530       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.overflowToDisk = false
15531       //  cache.name.edu_internet2_middleware_grouper_internal_dao_hib3_Hib3GroupDAO.statistics = false
15532   
15533       boolean madeChanges = false;
15534       
15535       if (maxElementsInMemory != null && !GrouperInstallerUtils.defaultString((String)grouperCacheBaseProperties.get("cache.name." + key + ".maxElementsInMemory")).equals(maxElementsInMemory.toString())) {
15536         grouperEhcachePropertiesContents.append("cache.name." + key + ".maxElementsInMemory = " + maxElementsInMemory + "\n");
15537         madeChanges = true;
15538       }
15539       if (eternal != null && !GrouperInstallerUtils.defaultString((String)grouperCacheBaseProperties.get("cache.name." + key + ".eternal")).equals(eternal.toString())) {
15540         grouperEhcachePropertiesContents.append("cache.name." + key + ".eternal = " + eternal + "\n");
15541         madeChanges = true;
15542       }
15543       if (timeToIdleSeconds != null && !GrouperInstallerUtils.defaultString((String)grouperCacheBaseProperties.get("cache.name." + key + ".timeToIdleSeconds")).equals(timeToIdleSeconds.toString())) {
15544         grouperEhcachePropertiesContents.append("cache.name." + key + ".timeToIdleSeconds = " + timeToIdleSeconds + "\n");
15545         madeChanges = true;
15546       }
15547       if (timeToLiveSeconds != null && !GrouperInstallerUtils.defaultString((String)grouperCacheBaseProperties.get("cache.name." + key + ".timeToLiveSeconds")).equals(timeToLiveSeconds.toString())) {
15548         grouperEhcachePropertiesContents.append("cache.name." + key + ".timeToLiveSeconds = " + timeToLiveSeconds + "\n");
15549         madeChanges = true;
15550       }
15551       if (overflowToDisk != null && !GrouperInstallerUtils.defaultString((String)grouperCacheBaseProperties.get("cache.name." + key + ".overflowToDisk")).equals(overflowToDisk.toString())) {
15552         grouperEhcachePropertiesContents.append("cache.name." + key + ".overflowToDisk = " + overflowToDisk + "\n");
15553         madeChanges = true;
15554       }
15555       if (statistics != null && !GrouperInstallerUtils.defaultString((String)grouperCacheBaseProperties.get("cache.name." + key + ".statistics")).equals(statistics.toString())) {
15556         grouperEhcachePropertiesContents.append("cache.name." + key + ".statistics = " + statistics + "\n");
15557         madeChanges = true;
15558       }
15559       if (madeChanges) {
15560         grouperEhcachePropertiesContents.append("\n");
15561       }
15562     }
15563   
15564     GrouperInstallerUtils.saveStringIntoFile(grouperCachePropertiesFile, grouperEhcachePropertiesContents.toString());
15565   }
15566 
15567   /**
15568    * get a subelement value.
15569    * e.g. if the node is &lt;source&gt;
15570    * and the sub element is &lt;id&gt;someId&lt;/id&gt;
15571    * It will return "someId" for subElementName "id"
15572    * @param parent
15573    * @param subElementName
15574    * @param required
15575    * @param descriptionForError 
15576    * @return the string or null if not there
15577    */
15578   public static String xmlElementValue(Element parent, String subElementName, boolean required, String descriptionForError) {
15579     
15580     NodeList nodeList = parent.getElementsByTagName(subElementName);
15581     
15582     if (nodeList.getLength() < 1) {
15583       if (required) {
15584         throw new RuntimeException("Cant find subElement <" + subElementName 
15585             + "> in parent element " + parent.getNodeName() + ", " + descriptionForError);
15586       }
15587       return null;
15588     }    
15589     
15590     if (nodeList.getLength() > 1) {
15591       throw new RuntimeException("Too many subElements <" + subElementName 
15592           + "> in parent element " + parent.getNodeName() + ", " 
15593           + nodeList.getLength() + ", " + descriptionForError);
15594     }
15595     return GrouperInstallerUtils.trimToEmpty(nodeList.item(0).getTextContent());
15596   }
15597   
15598   /**
15599    * put in a good comment about this param name
15600    * @param paramName
15601    * @param paramValue
15602    * @param subjectPropertiesContents
15603    */
15604   private static void convertSourcesXmlParamComment(String paramName, StringBuilder subjectPropertiesContents, String paramValue) {
15605     
15606     if (paramName == null) {
15607       throw new NullPointerException("param-name is null");
15608     }
15609     
15610     if (paramName.startsWith("subjectVirtualAttributeVariable_")) {
15611       subjectPropertiesContents.append("\n# when evaluating the virtual attribute EL expression, this variable can be used from this java class.\n"
15612           + "# " + paramName + " variable is the " + paramValue + " class.  Call static methods\n");
15613     } else if (paramName.startsWith("subjectVirtualAttribute_")) {
15614       
15615       Pattern pattern = Pattern.compile("^subjectVirtualAttribute_([\\d]+)_(.*)$");
15616       Matcher matcher = pattern.matcher(paramName);
15617       if (!matcher.matches()) {
15618         throw new RuntimeException(paramName + " is invalid, should be of form: subjectVirtualAttribute_<intIndex>_paramName");
15619       }
15620       
15621       String index = matcher.group(1);
15622       String attributeName = matcher.group(2);
15623       
15624       subjectPropertiesContents.append("\n# This virtual attribute index " + index + " is accessible via: subject.getAttributeValue(\"" + attributeName + "\");\n");
15625       
15626     } else if (GrouperInstallerUtils.equals(paramName, "findSubjectByIdOnCheckConfig")) {
15627       
15628       subjectPropertiesContents.append("\n# if a system check should try to resolve a subject by id on this source\n");
15629 
15630     } else if (GrouperInstallerUtils.equals(paramName, "subjectIdToFindOnCheckConfig")) {
15631       
15632       subjectPropertiesContents.append("\n# by default it will try to find a random string.  If you want a speicific ID to be found enter that here\n");
15633 
15634     } else if (GrouperInstallerUtils.equals(paramName, "findSubjectByIdentifiedOnCheckConfig")) {
15635       
15636       subjectPropertiesContents.append("\n# by default it will do a search by subject identifier\n");
15637 
15638     } else if (GrouperInstallerUtils.equals(paramName, "subjectIdentifierToFindOnCheckConfig")) {
15639       
15640       subjectPropertiesContents.append("\n# by default it will use a random value for subject identifier to lookup, you can specify a value here\n");
15641 
15642     } else if (GrouperInstallerUtils.equals(paramName, "findSubjectByStringOnCheckConfig")) {
15643       
15644       subjectPropertiesContents.append("\n# by default it will search for a subject by string\n");
15645 
15646     } else if (GrouperInstallerUtils.equals(paramName, "stringToFindOnCheckConfig")) {
15647       
15648       subjectPropertiesContents.append("\n# you can specify the search string here or it will be a random value\n");
15649 
15650     } else if (GrouperInstallerUtils.equals(paramName, "sortAttribute0")) {
15651       
15652       subjectPropertiesContents.append("\n# the 1st sort attribute for lists on screen that are derived from member table (e.g. search for member in group)\n"
15653           + "# you can have up to 5 sort attributes \n");
15654 
15655     } else if (GrouperInstallerUtils.equals(paramName, "sortAttribute1")) {
15656       
15657       subjectPropertiesContents.append("\n# the 2nd sort attribute for lists on screen that are derived from member table (e.g. search for member in group)\n"
15658           + "# you can have up to 5 sort attributes \n");
15659 
15660     } else if (GrouperInstallerUtils.equals(paramName, "sortAttribute2")) {
15661       
15662       subjectPropertiesContents.append("\n# the 3rd sort attribute for lists on screen that are derived from member table (e.g. search for member in group)\n"
15663           + "# you can have up to 5 sort attributes \n");
15664 
15665     } else if (GrouperInstallerUtils.equals(paramName, "sortAttribute3")) {
15666       
15667       subjectPropertiesContents.append("\n# the 4th sort attribute for lists on screen that are derived from member table (e.g. search for member in group)\n"
15668           + "# you can have up to 5 sort attributes \n");
15669 
15670     } else if (GrouperInstallerUtils.equals(paramName, "sortAttribute4")) {
15671       
15672       subjectPropertiesContents.append("\n# the 5th sort attribute for lists on screen that are derived from member table (e.g. search for member in group)\n"
15673           + "# you can have up to 5 sort attributes \n");
15674 
15675     } else if (GrouperInstallerUtils.equals(paramName, "searchAttribute0")) {
15676       
15677       subjectPropertiesContents.append("\n# the 1st search attribute for lists on screen that are derived from member table (e.g. search for member in group)\n"
15678           + "# you can have up to 5 search attributes \n");
15679 
15680     } else if (GrouperInstallerUtils.equals(paramName, "searchAttribute1")) {
15681       
15682       subjectPropertiesContents.append("\n# the 2nd search attribute for lists on screen that are derived from member table (e.g. search for member in group)\n"
15683           + "# you can have up to 5 search attributes \n");
15684 
15685     } else if (GrouperInstallerUtils.equals(paramName, "searchAttribute2")) {
15686       
15687       subjectPropertiesContents.append("\n# the 3rd search attribute for lists on screen that are derived from member table (e.g. search for member in group)\n"
15688           + "# you can have up to 5 search attributes \n");
15689 
15690     } else if (GrouperInstallerUtils.equals(paramName, "searchAttribute3")) {
15691       
15692       subjectPropertiesContents.append("\n# the 4th search attribute for lists on screen that are derived from member table (e.g. search for member in group)\n"
15693           + "# you can have up to 5 search attributes \n");
15694 
15695     } else if (GrouperInstallerUtils.equals(paramName, "searchAttribute4")) {
15696       
15697       subjectPropertiesContents.append("\n# the 5th search attribute for lists on screen that are derived from member table (e.g. search for member in group)\n"
15698           + "# you can have up to 5 search attributes\n");
15699 
15700     } else if (GrouperInstallerUtils.equals(paramName, "subjectIdentifierAttribute0")) {
15701       
15702       subjectPropertiesContents.append("\n# subject identifier to store in grouper's member table.  this is used to increase speed of loader and perhaps for provisioning\n"
15703           + "# you can have up to max 1 subject identifier\n");
15704 
15705     } else if (GrouperInstallerUtils.equals(paramName, "maxConnectionAge")) {
15706       
15707       subjectPropertiesContents.append("\n# seconds of max connection age\n");
15708 
15709     } else if (GrouperInstallerUtils.equals(paramName, "testConnectionOnCheckout")) {
15710       
15711       subjectPropertiesContents.append("\n# if connections from pool should be tested when checked out from pool\n");
15712 
15713     } else if (GrouperInstallerUtils.equals(paramName, "preferredTestQuery")) {
15714       
15715       subjectPropertiesContents.append("\n# query to use to test the connection when checking out from pool\n");
15716 
15717     } else if (GrouperInstallerUtils.equals(paramName, "idleConnectionTestPeriod")) {
15718       
15719       subjectPropertiesContents.append("\n# seconds between tests of idle connections in pool\n");
15720 
15721     } else if (GrouperInstallerUtils.equals(paramName, "dbDriver")) {
15722       
15723       subjectPropertiesContents.append("\n#       e.g. mysql:           com.mysql.jdbc.Driver\n"
15724           + "#       e.g. p6spy (log sql): com.p6spy.engine.spy.P6SpyDriver\n"
15725           + "#         for p6spy, put the underlying driver in spy.properties\n"
15726           + "#       e.g. oracle:          oracle.jdbc.driver.OracleDriver\n"
15727           + "#       e.g. postgres:        org.postgresql.Driver\n");
15728 
15729     } else if (GrouperInstallerUtils.equals(paramName, "dbUrl")) {
15730       
15731       subjectPropertiesContents.append("\n#       e.g. mysql:           jdbc:mysql://localhost:3306/grouper\n"
15732           + "#       e.g. p6spy (log sql): [use the URL that your DB requires]\n"
15733           + "#       e.g. oracle:          jdbc:oracle:thin:@server.school.edu:1521:sid\n"
15734           + "#       e.g. postgres:        jdbc:postgresql:grouper\n");
15735 
15736     } else if (GrouperInstallerUtils.equals(paramName, "dbUser")) {
15737       
15738       subjectPropertiesContents.append("\n# username when connecting to the database\n");
15739 
15740     } else if (GrouperInstallerUtils.equals(paramName, "dbPwd")) {
15741       
15742       subjectPropertiesContents.append("\n# password when connecting to the database (or file with encrypted password inside)\n");
15743 
15744     } else if (GrouperInstallerUtils.equals(paramName, "maxResults")) {
15745       
15746       subjectPropertiesContents.append("\n# maximum number of results from a search, generally no need to get more than 1000\n");
15747 
15748     } else if (GrouperInstallerUtils.equals(paramName, "dbTableOrView")) {
15749       
15750       subjectPropertiesContents.append("\n# the table or view to query results from.  Note, could prefix with a schema name\n");
15751 
15752     } else if (GrouperInstallerUtils.equals(paramName, "subjectIdCol")) {
15753       
15754       subjectPropertiesContents.append("\n# the column name to get the subjectId from\n");
15755 
15756     } else if (GrouperInstallerUtils.equals(paramName, "nameCol")) {
15757       
15758       subjectPropertiesContents.append("\n# the column name to get the name from\n");
15759 
15760     } else if (GrouperInstallerUtils.equals(paramName, "lowerSearchCol")) {
15761       
15762       subjectPropertiesContents.append("\n# search col where general searches take place, lower case\n");
15763 
15764     } else if (GrouperInstallerUtils.equals(paramName, "defaultSortCol")) {
15765       
15766       subjectPropertiesContents.append("\n# optional col if you want the search results sorted in the API (note, UI might override)\n");
15767 
15768     } else if (paramName.startsWith("subjectIdentifierCol")) {
15769       
15770       subjectPropertiesContents.append("\n# you can count up from 0 to N of columns to search by identifier (which might also include by id)\n");
15771 
15772     } else if (paramName.startsWith("subjectAttributeName")) {
15773       
15774       subjectPropertiesContents.append("\n# you can count up from 0 to N of attributes for various cols.  The name is how to reference in subject.getAttribute()\n");
15775 
15776     } else if (paramName.startsWith("subjectAttributeCol")) {
15777       
15778       subjectPropertiesContents.append("\n# now you can count up from 0 to N of attributes for various cols.  The name is how to reference in subject.getAttribute()\n");
15779 
15780     } else if (GrouperInstallerUtils.equals(paramName, "statusDatastoreFieldName")) {
15781       
15782       subjectPropertiesContents.append("\n# STATUS SECTION for searches to filter out inactives and allow\n"
15783           + "# the user to filter by status with e.g. status=all\n"
15784           + "# this is optional, and advanced\n"
15785           + "#\n"
15786           + "# field in database or ldap or endpoint that is the status field\n");
15787 
15788     } else if (GrouperInstallerUtils.equals(paramName, "statusLabel")) {
15789       
15790       subjectPropertiesContents.append("\n# search string from user which represents the status.  e.g. status=active\n");
15791 
15792     } else if (GrouperInstallerUtils.equals(paramName, "statusesFromUser")) {
15793       
15794       subjectPropertiesContents.append("\n# available statuses from screen (if not specified, any will be allowed). comma separated list.\n"
15795           + "# Note, this is optional and you probably dont want to configure it, it is mostly necessary\n"
15796           + "# when you have multiple sources with statuses...  if someone types an invalid status\n"
15797           + "# and you have this configured, it will not filter by it\n");
15798 
15799     } else if (GrouperInstallerUtils.equals(paramName, "statusAllFromUser")) {
15800       
15801       subjectPropertiesContents.append("\n# all label from the user\n");
15802 
15803     } else if (GrouperInstallerUtils.equals(paramName, "statusSearchDefault")) {
15804       
15805       subjectPropertiesContents.append("\n# if no status is specified, this will be used (e.g. for active only).  Note, the value should be of the\n"
15806           + "# form the user would type in\n");
15807 
15808     } else if (paramName.startsWith("statusTranslateUser")) {
15809       
15810       subjectPropertiesContents.append("\n# translate between screen values of status, and the data store value.  Increment the 0 to 1, 2, etc for more translations.\n"
15811           + "# so the user could enter: status=active, and that could translate to status_col=A.  The 'user' is what the user types in,\n"
15812           + "# the 'datastore' is what is in the datastore.  The user part is not case-sensitive.  Note, this could be a many to one\n");
15813 
15814     } else if (paramName.startsWith("statusTranslateDatastore")) {
15815       
15816       //hmmm, nothing to do here
15817     } else if (GrouperInstallerUtils.equals(paramName, "INITIAL_CONTEXT_FACTORY")) {
15818       
15819       subjectPropertiesContents.append("\n# e.g. com.sun.jndi.ldap.LdapCtxFactory\n");
15820 
15821     } else if (GrouperInstallerUtils.equals(paramName, "PROVIDER_URL")) {
15822       
15823       subjectPropertiesContents.append("\n# e.g. ldap://localhost:389\n");
15824 
15825     } else if (GrouperInstallerUtils.equals(paramName, "SECURITY_AUTHENTICATION")) {
15826       
15827       subjectPropertiesContents.append("\n# e.g. simple, none, sasl_mech\n");
15828 
15829     } else if (GrouperInstallerUtils.equals(paramName, "SECURITY_PRINCIPAL")) {
15830       
15831       subjectPropertiesContents.append("\n# e.g. cn=Manager,dc=example,dc=edu\n");
15832 
15833     } else if (GrouperInstallerUtils.equals(paramName, "SECURITY_CREDENTIALS")) {
15834       
15835       subjectPropertiesContents.append("\n# can be a password or a filename of the encrypted password\n");
15836 
15837     } else if (GrouperInstallerUtils.equals(paramName, "SubjectID_AttributeType")) {
15838       
15839       subjectPropertiesContents.append("\n# ldap attribute which is the subject id.  e.g. exampleEduRegID   Each subject has one and only one subject id.  Generally it is opaque and permanent.\n");
15840 
15841     } else if (GrouperInstallerUtils.equals(paramName, "SubjectID_formatToLowerCase")) {
15842       
15843       subjectPropertiesContents.append("\n# if the subject id should be changed to lower case after reading from datastore.  true or false\n");
15844 
15845     } else if (GrouperInstallerUtils.equals(paramName, "Name_AttributeType")) {
15846       
15847       subjectPropertiesContents.append("\n# attribute which is the subject name\n");
15848 
15849     } else if (GrouperInstallerUtils.equals(paramName, "Description_AttributeType")) {
15850       
15851       subjectPropertiesContents.append("\n# attribute which is the subject description\n");
15852 
15853     } else if (GrouperInstallerUtils.equals(paramName, "VTLDAP_VALIDATOR")) {
15854       
15855       subjectPropertiesContents.append("\n# LdapValidator provides an interface for validating ldap objects when they are in the pool.\n"
15856           + "# ConnectLdapValidator validates an ldap connection is healthy by testing it is connected.\n"
15857           + "# CompareLdapValidator validates an ldap connection is healthy by performing a compare operation.\n");
15858 
15859     } else if (GrouperInstallerUtils.equals(paramName, "VTLDAP_VALIDATOR_COMPARE_DN")) {
15860       
15861       subjectPropertiesContents.append("\n# if VTLDAP_VALIDATOR is CompareLdapValidator, this is the DN of the ldap object to get, e.g. ou=People,dc=vt,dc=edu\n");
15862 
15863     } else if (GrouperInstallerUtils.equals(paramName, "VTLDAP_VALIDATOR_COMPARE_SEARCH_FILTER_STRING")) {
15864       
15865       subjectPropertiesContents.append("\n# if VTLDAP_VALIDATOR is CompareLdapValidator, this is the filter string, e.g. ou=People\n");
15866 
15867     } else {
15868       
15869       //hmmm, not sure what to do here, no comment
15870       subjectPropertiesContents.append("\n");
15871 
15872     }
15873 
15874   }
15875   
15876   /**
15877    * valid source param pattern
15878    */
15879   private static Pattern sourcesValidParamPattern = Pattern.compile("^[A-Za-z0-9_]+$");
15880   
15881   /**
15882    * 
15883    * @param subjectPropertiesFile 
15884    * @param sourcesXmlUrl
15885    */
15886   public static void convertSourcesXmlToProperties(File subjectPropertiesFile, URL sourcesXmlUrl) {
15887 
15888     //look at base properties
15889     Properties subjectProperties = subjectPropertiesFile.exists() ? 
15890         GrouperInstallerUtils.propertiesFromFile(subjectPropertiesFile) : new Properties();
15891 
15892     if (subjectPropertiesFile.exists()) {
15893       
15894       //lets see if it just has the default.  the default has no properties
15895       if (subjectProperties.size() > 0) {
15896         throw new RuntimeException(subjectPropertiesFile.getAbsolutePath() + " exists and must not!  Backup your subject.properties and run this again and merge your subject.properties into the result");
15897       }
15898     }
15899     
15900     if (!subjectPropertiesFile.getParentFile().exists() || !subjectPropertiesFile.getParentFile().isDirectory()) {
15901       throw new RuntimeException(subjectPropertiesFile.getParentFile().getAbsolutePath() + " must exist and must be a directory");
15902     }
15903     
15904     StringBuilder subjectPropertiesContents = new StringBuilder();
15905     
15906     subjectPropertiesContents.append(
15907               "# Copyright 2016 Internet2\n"
15908             + "#\n"
15909             + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n"
15910             + "# you may not use this file except in compliance with the License.\n"
15911             + "# You may obtain a copy of the License at\n"
15912             + "#\n"
15913             + "#   http://www.apache.org/licenses/LICENSE-2.0\n"
15914             + "#\n"
15915             + "# Unless required by applicable law or agreed to in writing, software\n"
15916             + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n"
15917             + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n"
15918             + "# See the License for the specific language governing permissions and\n"
15919             + "# limitations under the License.\n"
15920             + "\n"
15921             + "#\n"
15922             + "# Subject configuration\n"
15923             + "#\n"
15924             + "\n"
15925             + "# The subject properties uses Grouper Configuration Overlays (documented on wiki)\n"
15926             + "# By default the configuration is read from subject.base.properties\n"
15927             + "# (which should not be edited), and the subject.properties overlays\n"
15928             + "# the base settings.  See the subject.base.properties for the possible\n"
15929             + "# settings that can be applied to the subject.properties\n\n"
15930         );
15931 
15932     subjectPropertiesContents.append(
15933         "# enter the location of the sources.xml.  Must start with classpath: or file:\n"
15934         + "# blank means dont use sources.xml, use subject.properties\n"
15935         + "# default is: classpath:sources.xml\n"
15936         + "# e.g. file:/dir1/dir2/sources.xml\n"
15937         + "subject.sources.xml.location = \n\n");
15938       
15939     //   <source adapterClass="edu.internet2.middleware.grouper.GrouperSourceAdapter">
15940     NodeList sourcesNodeList = GrouperInstallerUtils.xpathEvaluate(sourcesXmlUrl, "/sources/source");
15941     
15942     Set<String> usedConfigNames = new HashSet<String>();
15943     
15944     for (int i=0;i<sourcesNodeList.getLength();i++) {
15945 
15946       Element sourceElement = (Element)sourcesNodeList.item(i);
15947       
15948       String configName = null;
15949       String id = null;
15950       {
15951         //  #########################################
15952         //  ## Configuration for source: whateverId
15953         //  #########################################
15954         //  # generally the <configName> is the same as or similar to the source id.  This cannot have special characters
15955         //  # this links together all the configs for this source
15956         //  # subjectApi.source.<configName>.id = sourceId
15957         id = xmlElementValue(sourceElement, "id", true, "source index " + i);
15958         
15959         //these are configured in subject.base.properties
15960         if (GrouperInstallerUtils.equals(id, "g:gsa")
15961             || GrouperInstallerUtils.equals(id, "grouperEntities")) {
15962           continue;
15963         }
15964         configName = convertEhcacheNameToPropertiesKey(id, usedConfigNames);
15965         usedConfigNames.add(configName);
15966         
15967         subjectPropertiesContents.append(
15968             "\n#########################################\n"
15969             + "## Configuration for source id: " + id + "\n"
15970             + "## Source configName: " + configName + "\n"
15971             + "#########################################\n"
15972             + "subjectApi.source." + configName + ".id = " + id + "\n"
15973             );
15974       }
15975 
15976       {
15977         // <name>Grouper: Group Source Adapter</name>
15978         String name = xmlElementValue(sourceElement, "name", true, "source: " + id);
15979         subjectPropertiesContents.append("\n# this is a friendly name for the source\n"
15980             + "subjectApi.source." + configName + ".name = " + name + "\n");
15981       }
15982       
15983       {
15984         // <type>group</type>
15985         NodeList typeNodeList = sourceElement.getElementsByTagName("type");
15986         Set<String> typeSet = new LinkedHashSet<String>();
15987         
15988         for (int typeIndex=0; typeIndex<typeNodeList.getLength(); typeIndex++) {
15989           
15990           typeSet.add(GrouperInstallerUtils.trimToEmpty(typeNodeList.item(typeIndex).getTextContent()));
15991           
15992         }
15993         if (typeNodeList.getLength() > 0) {
15994           
15995           subjectPropertiesContents.append("\n# type is not used all that much.  Can have multiple types, comma separate.  Can be person, group, application\n"
15996               + "subjectApi.source." + configName + ".types = " + GrouperInstallerUtils.join(typeSet.iterator(), ", ") + "\n"
15997               );
15998         }
15999 
16000       }
16001 
16002       {
16003         NamedNodeMap configuredNamedNodeMap = sourceElement.getAttributes();
16004         if (configuredNamedNodeMap.getLength() != 1 || !"adapterClass".equals(configuredNamedNodeMap.item(0).getNodeName())) {
16005           throw new RuntimeException("Expecting one source attribute: adapterClass for source: " + id);
16006         }
16007         
16008         String adapterClass = sourceElement.getAttribute("adapterClass");
16009 
16010         subjectPropertiesContents.append("\n# the adapter class implements the interface: edu.internet2.middleware.subject.Source\n");
16011         subjectPropertiesContents.append("# adapter class must extend: edu.internet2.middleware.subject.provider.BaseSourceAdapter\n");
16012         subjectPropertiesContents.append("# edu.internet2.middleware.grouper.subj.GrouperJdbcSourceAdapter2  :  if doing JDBC this should be used if possible.  All subject data in one table/view.\n");
16013         subjectPropertiesContents.append("# edu.internet2.middleware.grouper.subj.GrouperJdbcSourceAdapter   :  oldest JDBC source.  Put freeform queries in here\n");
16014         subjectPropertiesContents.append("# edu.internet2.middleware.grouper.subj.GrouperJndiSourceAdapter   :  used for LDAP\n");
16015         subjectPropertiesContents.append("subjectApi.source." + configName + ".adapterClass = " + adapterClass + "\n");
16016       }      
16017 
16018       //  # You can flag a source as not throwing exception on a findAll (general search) i.e. if it is
16019       //  # ok if it is down.  Generally you probably won't want to do this.  It defaults to true if omitted.
16020       //  # subjectApi.source.<configName>.param.throwErrorOnFindAllFailure.value = true
16021 
16022       {
16023         //  <init-param>
16024         //    <param-name>subjectVirtualAttribute_0_searchAttribute0</param-name>
16025         //    <param-value>${subject.getAttributeValue('name')},${subject.getAttributeValue('displayName')},${subject.getAttributeValue('alternateName')}</param-value>
16026         //  </init-param>
16027         NodeList initParamNodeList = sourceElement.getElementsByTagName("init-param");
16028         
16029         Set<String> usedParamNames = new HashSet<String>();
16030 
16031         for (int initParamIndex=0; initParamIndex<initParamNodeList.getLength(); initParamIndex++) {
16032           
16033           Element initParamElement = (Element)initParamNodeList.item(initParamIndex);
16034           String paramName = xmlElementValue(initParamElement, "param-name", true, "param-name index " + initParamIndex + " in source " + id);
16035           String paramValue = xmlElementValue(initParamElement, "param-value", true, "param-value " + paramName + " in source " + id);
16036           
16037           String paramConfigKey = convertEhcacheNameToPropertiesKey(paramName, usedParamNames);
16038           convertSourcesXmlParamComment(paramName, subjectPropertiesContents, paramValue);
16039 
16040           //if the param name is invalid, then have a name config
16041           if (!GrouperInstallerUtils.equals(paramName, paramConfigKey)) {
16042             subjectPropertiesContents.append("subjectApi.source." + configName + ".param." + paramConfigKey + ".name = " + paramName + "\n");
16043           }
16044           
16045           //cant have newlines in there, convert to spaces
16046           paramValue = GrouperInstallerUtils.replaceNewlinesWithSpace(paramValue);
16047           
16048           //  # subjectApi.source.<configName>.param.throwErrorOnFindAllFailure.value = true
16049           subjectPropertiesContents.append("subjectApi.source." + configName + ".param." + paramConfigKey + ".value = " + paramValue + "\n");
16050 
16051         }
16052 
16053       }
16054 
16055       {
16056         //  <search>
16057         //    <searchType>searchSubject</searchType>
16058         //    <param>
16059         //      <param-name>sql</param-name>
16060         //                <param-value>
16061         //                  select
16062         //                    s.subjectid as id, s.name as name,
16063         //      </param-value>
16064         //    </param>
16065         //  </search>
16066         NodeList searchNodeList = sourceElement.getElementsByTagName("search");
16067         
16068         for (int searchIndex=0; searchIndex<searchNodeList.getLength(); searchIndex++) {
16069           
16070           Element searchElement = (Element)searchNodeList.item(searchIndex);
16071           
16072           String searchType = xmlElementValue(searchElement, "searchType", true, "search element in the source: " + id);
16073 
16074           NodeList searchParamNodeList = searchElement.getElementsByTagName("param");
16075 
16076           if (GrouperInstallerUtils.equals(searchType, "searchSubject")) {
16077             subjectPropertiesContents.append("\n#searchSubject: find a subject by ID.  ID is generally an opaque and permanent identifier, e.g. 12345678.\n"
16078                 + "#  Each subject has one and only on ID.  Returns one result when searching for one ID.\n");
16079           } else if (GrouperInstallerUtils.equals(searchType, "searchSubjectByIdentifier")) {
16080             subjectPropertiesContents.append("\n#searchSubjectByIdentifier: find a subject by identifier.  Identifier is anything that uniquely\n"
16081                 + "#  identifies the user, e.g. jsmith or jsmith@institution.edu.\n"
16082                 + "#  Subjects can have multiple identifiers.  Note: it is nice to have if identifiers are unique\n"
16083                 + "#  even across sources.  Returns one result when searching for one identifier.\n");
16084           } else if (GrouperInstallerUtils.equals(searchType, "search")) {
16085             subjectPropertiesContents.append("\n#   search: find subjects by free form search.  Returns multiple results.\n");
16086           } else {
16087             System.out.println("Not expecting searchType: '" + searchType + "'");
16088           }
16089 
16090           for (int searchParamIndex=0; searchParamIndex<searchParamNodeList.getLength(); searchParamIndex++) {
16091             
16092             Element searchParamElement = (Element)searchParamNodeList.item(searchParamIndex);
16093             
16094             String paramName = xmlElementValue(searchParamElement, "param-name", true, 
16095                 "search param name element index " + searchParamIndex + " in the source: " + id);
16096           
16097             String paramValue = xmlElementValue(searchParamElement, "param-value", true, 
16098                 "search param value element index " + searchParamIndex + " in the source: " + id);
16099 
16100             // cant have newlines in a properties file
16101             paramValue = GrouperInstallerUtils.replaceNewlinesWithSpace(paramValue);
16102 
16103             //  #
16104             //  # This is how search params are specified.  Note, each source can have different params for each search type
16105             //  # subjectApi.source.<configName>.search.<searchType>.param.<paramName>.value = something
16106             //  #
16107             //  ##############################################
16108             //  #
16109             //  # Searches for edu.internet2.middleware.grouper.subj.GrouperJdbcConnectionProvider
16110             //  #
16111             //  # searchSubject:
16112             //  #
16113             //  # subjectApi.source.<configName>.search.searchSubject.param.sql.value = select s.subjectid as id, s.name as name, (select sa2.value from subjectattribute sa2 where name='name' and sa2.SUBJECTID = s.subjectid) as lfname, (select sa3.value from subjectattribute sa3 where name='loginid' and sa3.SUBJECTID = s.subjectid) as loginid, (select sa4.value from subjectattribute sa4 where name='description' and sa4.SUBJECTID = s.subjectid) as description, (select sa5.value from subjectattribute sa5 where name='email' and sa5.SUBJECTID = s.subjectid) as email from subject s where {inclause}
16114             //  #    inclause allows searching by subject id for multiple ids in one query
16115             //  # subjectApi.source.<configName>.search.searchSubject.param.inclause.value = s.subjectid = ?
16116 
16117             if (!sourcesValidParamPattern.matcher(paramName).matches()) {
16118               throw new RuntimeException("Source " + id + " search " + searchType + " param name is not valid: '" + paramName + "'");
16119             }
16120             if (GrouperInstallerUtils.equals(searchType, "searchSubject")) {
16121               if (GrouperInstallerUtils.equals("sql", paramName)) {
16122                 subjectPropertiesContents.append("\n# sql is the sql to search for the subject by id should use an {inclause}\n");
16123               } else if (GrouperInstallerUtils.equals("inclause", paramName)) {
16124                 subjectPropertiesContents.append("\n# inclause allows searching by subject for multiple ids or identifiers in one query, must have {inclause} in the sql query,\n"
16125                     + "#    this will be subsituted to in clause with the following.  Should use a question mark ? for bind variable\n");
16126               } else if (GrouperInstallerUtils.equals("filter", paramName)) {
16127                   subjectPropertiesContents.append("\n# sql is the sql to search for the subject by id.  %TERM% will be subsituted by the id searched for\n");
16128               }
16129             } else if (GrouperInstallerUtils.equals(searchType, "searchSubjectByIdentifier")) {
16130               if (GrouperInstallerUtils.equals("sql", paramName)) {
16131                 subjectPropertiesContents.append("\n# sql is the sql to search for the subject by identifier should use an {inclause}\n");
16132               } else if (GrouperInstallerUtils.equals("inclause", paramName)) {
16133                 subjectPropertiesContents.append("\n# inclause allows searching by subject for multiple ids or identifiers in one query, must have {inclause} in the sql query,\n"
16134                     + "#    this will be subsituted to in clause with the following.  Should use a question mark ? for bind variable\n");
16135               } else if (GrouperInstallerUtils.equals("filter", paramName)) {
16136                 subjectPropertiesContents.append("\n# sql is the sql to search for the subject by identifier.  %TERM% will be subsituted by the identifier searched for\n");
16137               }
16138             } else if (GrouperInstallerUtils.equals(searchType, "search")) {
16139               if (GrouperInstallerUtils.equals("sql", paramName)) {
16140                 subjectPropertiesContents.append("\n# sql is the sql to search for the subject free-form search.  user question marks for bind variables\n");
16141               } else if (GrouperInstallerUtils.equals("inclause", paramName)) {
16142                 throw new RuntimeException("Should not have incluse for search of type search in source: " + id);
16143               } else if (GrouperInstallerUtils.equals("filter", paramName)) {
16144                 subjectPropertiesContents.append("\n# sql is the sql to search for the subject by free form search.  %TERM% will be subsituted by the text searched for\n");
16145               }
16146             }
16147             if (GrouperInstallerUtils.equals("scope", paramName)) {
16148               subjectPropertiesContents.append("\n# Scope Values can be: OBJECT_SCOPE, ONELEVEL_SCOPE, SUBTREE_SCOPE\n");
16149             } else if (GrouperInstallerUtils.equals("base", paramName)) {
16150               subjectPropertiesContents.append("\n# base dn to search in\n");
16151             }
16152             
16153             //  # subjectApi.source.<configName>.search.<searchType>.param.<paramName>.value = something
16154             subjectPropertiesContents.append("subjectApi.source." + configName + ".search." + searchType + ".param." + paramName + ".value = " + paramValue + "\n");
16155             
16156           }
16157         }
16158       }
16159       
16160       {
16161         // # attributes from ldap object to become subject attributes.  comma separated
16162         // <attribute>cn</attribute>
16163         // <attribute>sn</attribute>
16164         NodeList attributeNodeList = sourceElement.getElementsByTagName("attribute");
16165         Set<String> attributeSet = new LinkedHashSet<String>();
16166 
16167         for (int attributeIndex=0; attributeIndex<attributeNodeList.getLength(); attributeIndex++) {
16168 
16169           attributeSet.add(GrouperInstallerUtils.trimToEmpty(attributeNodeList.item(attributeIndex).getTextContent()));
16170         }
16171         if (attributeNodeList.getLength() > 0) {
16172 
16173           subjectPropertiesContents.append("\n# attributes from ldap object to become subject attributes.  comma separated\n"
16174               + "subjectApi.source." + configName + ".attributes = " + GrouperInstallerUtils.join(attributeSet.iterator(), ", ") + "\n");
16175 
16176         }
16177 
16178       }
16179       
16180       {
16181         // # internal attributes are used by grouper only not exposed to code that uses subjects.  comma separated
16182         // <internal-attributes>cn</internal-attributes>
16183         // <internal-attributes>sn</internal-attributes>
16184         NodeList internalAttributeNodeList = sourceElement.getElementsByTagName("internal-attribute");
16185         Set<String> internalAttributeSet = new LinkedHashSet<String>();
16186 
16187         for (int internalAttributeIndex=0; internalAttributeIndex<internalAttributeNodeList.getLength(); internalAttributeIndex++) {
16188 
16189           internalAttributeSet.add(GrouperInstallerUtils.trimToEmpty(internalAttributeNodeList.item(internalAttributeIndex).getTextContent()));
16190 
16191         }
16192         if (internalAttributeNodeList.getLength() > 0) {
16193 
16194           subjectPropertiesContents.append("\n# internal attributes are used by grouper only not exposed to code that uses subjects.  comma separated\n"
16195               + "subjectApi.source." + configName + ".internalAttributes = " + GrouperInstallerUtils.join(internalAttributeSet.iterator(), ", ") + "\n");
16196 
16197         }
16198 
16199       }
16200       //space between sources
16201       subjectPropertiesContents.append("\n");
16202     }
16203 
16204     GrouperInstallerUtils.saveStringIntoFile(subjectPropertiesFile, subjectPropertiesContents.toString());
16205 
16206   }
16207 
16208   /**
16209    * edit an xml file attribute in a xml file
16210    * @param file
16211    * @param elementName
16212    * @param elementMustHaveAttributeAndValue
16213    * @param newValue
16214    * @param description of change for sys out print
16215    * @param newAttributeName if adding new attribute, this is the name
16216    * @return true if edited file, or false if not but didnt need to, null if not found
16217    */
16218   public static Boolean editXmlFileAttribute(File file, String elementName, Map<String, String> elementMustHaveAttributeAndValue, 
16219       String newAttributeName, String newValue, String description) {
16220 
16221     if (!file.exists() || file.length() == 0) {
16222       throw new RuntimeException("Why does " + file.getName() + " not exist and have contents? " 
16223           + file.getAbsolutePath());
16224     }
16225     
16226     String fileContents = GrouperInstallerUtils.readFileIntoString(file);
16227     
16228     boolean inComment = false;
16229     
16230     //lets parse the file and get to the element
16231     OUTER: for (int i=0;i<fileContents.length();i++) {
16232       
16233       //look for start element
16234       char curChar = fileContents.charAt(i);
16235 
16236       Character nextChar = (i+1) < fileContents.length() ? fileContents.charAt(i+1) : null;
16237       Character nextNextChar = (i+2) < fileContents.length() ? fileContents.charAt(i+2) : null;
16238       Character nextNextNextChar = (i+3) < fileContents.length() ? fileContents.charAt(i+3) : null;
16239       
16240       //if we are in comment, see when we are out of comment
16241       if (inComment) {
16242         if (curChar == '-' && nextChar != null && nextChar == '-' && nextNextChar != null && nextNextChar == '>') {
16243           inComment = false;
16244         }
16245         continue;
16246         
16247       }
16248 
16249       //look for a tag or comment
16250       if (curChar != '<') {
16251         continue;
16252       }
16253       
16254       //see if this is a comment
16255       if (nextChar != null && nextChar == '!' && nextNextChar != null && nextNextChar == '-' && nextNextNextChar != null && nextNextNextChar == '-') {
16256         inComment = true;
16257         continue;
16258       }
16259 
16260       //get tagName
16261       String currentElementName = _internalXmlTagName(fileContents, i+1);
16262       
16263       //not the right tag
16264       if (!GrouperInstallerUtils.equals(currentElementName, elementName)) {
16265         continue;
16266       }
16267       
16268       int tagNameStart = fileContents.indexOf(currentElementName, i+1);
16269       
16270       //get the attributes
16271       int tagAttributesStart = tagNameStart + currentElementName.length();
16272       XmlParseAttributesResult xmlParseAttributesResult = _internalXmlParseAttributes(fileContents, tagAttributesStart);
16273       Map<String, String> currentAttributes = xmlParseAttributesResult.getAttributes();
16274       
16275       if (GrouperInstallerUtils.length(elementMustHaveAttributeAndValue) > 0) {
16276         for (String attributeName : elementMustHaveAttributeAndValue.keySet()) {
16277           String expectedValue = elementMustHaveAttributeAndValue.get(attributeName);
16278           String hasValue = currentAttributes.get(attributeName);
16279 
16280           //if we dont have that value, then keep going
16281           if (!GrouperInstallerUtils.equals(expectedValue, hasValue)) {
16282             continue OUTER;
16283           }
16284         }
16285       }
16286       
16287       //we have the tag and it has the expected attributes
16288 
16289       //see if the attribute is even there...
16290       if (!currentAttributes.containsKey(newAttributeName)) {
16291         System.out.println(" - adding " + description + " with value: '" + newValue + "'");
16292         String newFileContents = fileContents.substring(0, tagAttributesStart) + " " + newAttributeName + "=\"" + newValue + 
16293             "\" " + fileContents.substring(tagAttributesStart, fileContents.length());
16294         GrouperInstallerUtils.writeStringToFile(file, newFileContents);
16295         return true;
16296       }
16297 
16298       //does it already have the value?
16299       String currentValue = currentAttributes.get(newAttributeName);
16300       
16301       //value is already there
16302       if (GrouperInstallerUtils.equals(currentValue, newValue)) {
16303         return false;
16304       }
16305 
16306       //it has the wrong value
16307       int startQuote = xmlParseAttributesResult.getAttributeStartIndex().get(newAttributeName);
16308       int endQuote = xmlParseAttributesResult.getAttributeEndIndex().get(newAttributeName);
16309 
16310       System.out.println(" - changing " + description + " from old value: '" + currentValue 
16311           + "' to new value: '" + newValue + "'");
16312 
16313       String newFileContents = fileContents.substring(0, startQuote+1)  + newValue + 
16314           fileContents.substring(endQuote, fileContents.length());
16315       GrouperInstallerUtils.writeStringToFile(file, newFileContents);
16316       return true;
16317 
16318     }
16319 
16320     return null;
16321 
16322   }
16323 
16324   /**
16325    * 
16326    * @param fileContents
16327    * @param tagIndexStart
16328    * @return the tag name
16329    */
16330   private static String _internalXmlTagName(String fileContents, int tagIndexStart) {
16331     StringBuilder tagName = new StringBuilder();
16332     for (int i=tagIndexStart; i<fileContents.length(); i++) {
16333       char curChar = fileContents.charAt(i);
16334       if (tagName.length() == 0 && Character.isWhitespace(curChar)) {
16335         continue;
16336       }
16337       if (Character.isWhitespace(curChar) || '/' == curChar || '>' == curChar) {
16338         return tagName.toString();
16339       }
16340       tagName.append(curChar);
16341     }
16342     throw new RuntimeException("How did I get here???? '" + tagName.toString() + "'");
16343   }
16344   
16345   /**
16346    * xml parse attribute result
16347    */
16348   private static class XmlParseAttributesResult {
16349 
16350     /**
16351      * attributes name to value
16352      */
16353     private Map<String, String> attributes;
16354     
16355     /**
16356      * attribute name to startIndex (of quote)
16357      */
16358     private Map<String, Integer> attributeStartIndex;
16359 
16360     /**
16361      * attribute name to endIndex (of quote)
16362      */
16363     private Map<String, Integer> attributeEndIndex;
16364 
16365     
16366     /**
16367      * attributes name to value
16368      * @return the attributes
16369      */
16370     public Map<String, String> getAttributes() {
16371       return this.attributes;
16372     }
16373 
16374     
16375     /**
16376      * attributes name to value
16377      * @param attributes1 the attributes to set
16378      */
16379     public void setAttributes(Map<String, String> attributes1) {
16380       this.attributes = attributes1;
16381     }
16382 
16383     
16384     /**
16385      * attribute name to startIndex (of quote)
16386      * @return the attributeStartIndex
16387      */
16388     public Map<String, Integer> getAttributeStartIndex() {
16389       return this.attributeStartIndex;
16390     }
16391     
16392     /**
16393      * attribute name to startIndex (of quote)
16394      * @param attributeStartIndex1 the attributeStartIndex to set
16395      */
16396     public void setAttributeStartIndex(Map<String, Integer> attributeStartIndex1) {
16397       this.attributeStartIndex = attributeStartIndex1;
16398     }
16399     
16400     /**
16401      * attribute name to endIndex (of quote)
16402      * @return the attributeEndIndex
16403      */
16404     public Map<String, Integer> getAttributeEndIndex() {
16405       return this.attributeEndIndex;
16406     }
16407     
16408     /**
16409      * attribute name to endIndex (of quote)
16410      * @param attributeEndIndex1 the attributeEndIndex to set
16411      */
16412     public void setAttributeEndIndex(Map<String, Integer> attributeEndIndex1) {
16413       this.attributeEndIndex = attributeEndIndex1;
16414     }
16415     
16416   }
16417 
16418   /**
16419    * 
16420    */
16421   public static enum GrouperInstallerAdminManageServiceAction {
16422   
16423     /** start */
16424     start,
16425     
16426     /** stop */
16427     stop,
16428     
16429     /** restart */
16430     restart,
16431     
16432     /** status */
16433     status;
16434     
16435     /**
16436      * 
16437      * @param string
16438      * @param exceptionIfInvalid
16439      * @param exceptionIfBlank
16440      * @return the action
16441      */
16442     public static GrouperInstallerAdminManageServiceAction valueOfIgnoreCase(String string, boolean exceptionIfBlank, boolean exceptionIfInvalid) {
16443       return GrouperInstallerUtils.enumValueOfIgnoreCase(GrouperInstallerAdminManageServiceAction.class, string, exceptionIfBlank, exceptionIfInvalid);
16444     }
16445     
16446   }
16447 
16448   /**
16449    * parse attributes
16450    * @param fileContents
16451    * @param tagAttributesStart is the index where the attributes start
16452    * @return the map of attribute names and values
16453    */
16454   private static XmlParseAttributesResult _internalXmlParseAttributes(String fileContents, int tagAttributesStart) {
16455 
16456     XmlParseAttributesResult xmlParseAttributesResult = new XmlParseAttributesResult();
16457 
16458     Map<String, String> attributes = new LinkedHashMap<String, String>();
16459     Map<String, Integer> attributeStartIndex = new LinkedHashMap<String, Integer>();
16460     Map<String, Integer> attributeEndIndex = new LinkedHashMap<String, Integer>();
16461 
16462     xmlParseAttributesResult.setAttributes(attributes);
16463     xmlParseAttributesResult.setAttributeStartIndex(attributeStartIndex);
16464     xmlParseAttributesResult.setAttributeEndIndex(attributeEndIndex);
16465     
16466     boolean inAttributeStartValue = false;
16467     boolean inAttributeStartName = true;
16468     boolean inAttributeName = false;
16469     boolean inAttributeValue = false;
16470     
16471     StringBuilder attributeName = null;
16472     StringBuilder attributeValue = null;
16473     
16474     for (int i=tagAttributesStart; i<fileContents.length(); i++) {
16475       char curChar = fileContents.charAt(i);
16476       boolean isWhitespace = Character.isWhitespace(curChar);
16477 
16478       //waiting for the attribute
16479       if ((inAttributeStartValue || inAttributeStartName) && isWhitespace) {
16480         continue;
16481       }
16482 
16483       //if waiting for value and equals, keep looking
16484       if (inAttributeStartValue && curChar == '=') {
16485         continue;
16486       }
16487 
16488       //waiting to start an attribute name, its not whitespace so do it
16489       if (inAttributeStartName) {
16490         
16491         //we done if we got to this character
16492         if (curChar == '/' || curChar == '>') {
16493           return xmlParseAttributesResult;
16494         }
16495         
16496         inAttributeStartName = false;
16497         inAttributeName = true;
16498         attributeName = new StringBuilder();
16499       }
16500 
16501       //if in an attribute name and whitespace or equals, then we are done
16502       if (inAttributeName && (isWhitespace || curChar == '=' )) {
16503         inAttributeName = false;
16504         inAttributeStartValue = true;
16505         continue;
16506       }
16507 
16508       //getting the attribute name
16509       if (inAttributeName) {
16510         attributeName.append(curChar);
16511         continue;
16512       }
16513 
16514       //if waiting for start value and found quote
16515       if (inAttributeStartValue && curChar == '"') {
16516         inAttributeStartValue = false;
16517         inAttributeValue = true;
16518         attributeValue = new StringBuilder();
16519         attributeStartIndex.put(attributeName.toString(), i);
16520         continue;
16521       }
16522 
16523       //if in attribute value and not quote, append the char
16524       if (inAttributeValue && curChar != '"') {
16525         attributeValue.append(curChar);
16526         continue;
16527       }
16528 
16529       //done with attribute value
16530       if (inAttributeValue && curChar == '"') {
16531         inAttributeValue = false;
16532         inAttributeStartName = true;
16533         if (attributes.containsKey(attributeName.toString())) {
16534           throw new RuntimeException("Duplicate attribute: " + attributeName.toString());
16535         }
16536         attributes.put(attributeName.toString(), attributeValue.toString());
16537         attributeEndIndex.put(attributeName.toString(), i);
16538         continue;
16539       }
16540 
16541       throw new RuntimeException("Why are we here? " + i + ", " + fileContents);
16542     }
16543     return xmlParseAttributesResult;
16544   }
16545   
16546   /** revert patch excludes */
16547   private static Set<String> revertPatchExcludes = new HashSet<String>();
16548   
16549   static {
16550     revertPatchExcludes.add("grouper.cache.properties");
16551     revertPatchExcludes.add("ehcache.xml");
16552     revertPatchExcludes.add("ehcache.example.xml");
16553   }
16554 }