@Override public ServletOutputStream getOutputStream() throws IOException { return new ServletOutputStream() { @Override public boolean isReady() { return false; } @Override public void setWriteListener(WriteListener writeListener) { } private TeeOutputStream tee = new TeeOutputStream(ResponseWrapper.super.getOutputStream(), bos); @Override public void write(int b) throws IOException { tee.write(b); } }; }
/** * Returns an OuputStream that writes to local webapp and to file inside <code>MCR.WCMS2.DataDir</code>. */ public static OutputStream getOutputStream(String path) throws IOException { String cleanPath = path.startsWith("/") ? path.substring(1) : path; File wcmsDataDir = getWCMSDataDir(); File webappBaseDir = getWebAppBaseDir(); File webappTarget = new File(webappBaseDir, cleanPath); if (!webappTarget.toPath().startsWith(webappBaseDir.toPath())) { throw new IOException(String.format(Locale.ROOT, "Cannot write %s outside the web application: %s", webappTarget, webappBaseDir)); } File wcmsDataDirTarget = new File(wcmsDataDir, cleanPath); createDirectoryIfNeeded(webappTarget); createDirectoryIfNeeded(wcmsDataDirTarget); LOGGER.info(String.format(Locale.ROOT, "Writing content to %s and to %s.", webappTarget, wcmsDataDirTarget)); return new TeeOutputStream(new FileOutputStream(wcmsDataDirTarget), new FileOutputStream(webappTarget)); }
private static void startPlatformLogging(Path fileName) { sysOut = System.out; sysErr = System.err; try { File file = null; file = fileName.toFile(); file.getParentFile().mkdirs(); file.createNewFile(); FileOutputStream fos = new FileOutputStream(file); TeeOutputStream bothStream =new TeeOutputStream(System.out, fos); PrintStream ps = new PrintStream(bothStream); System.setOut(ps); System.setErr(ps); } catch(Exception e) { e.printStackTrace(); throw new IllegalArgumentException("cannot redirect to output file"); } System.out.println("StartTime: " + System.currentTimeMillis()); }
public AbstractCppcheckCommand(IConsole console, String[] defaultArguments, long timeout, String binaryPath) { this.binaryPath = binaryPath; this.console = console; this.defaultArguments = defaultArguments; executor = new DefaultExecutor(); // all modes of operation returns 0 when no error occured, executor.setExitValue(0); watchdog = new ExecuteWatchdog(timeout); executor.setWatchdog(watchdog); out = new ByteArrayOutputStream(); err = new ByteArrayOutputStream(); processStdOut = new LineFilterOutputStream(new TeeOutputStream(out, console.getConsoleOutputStream(false)), DEFAULT_CHARSET); processStdErr = new LineFilterOutputStream(new TeeOutputStream(err, console.getConsoleOutputStream(true)), DEFAULT_CHARSET); }
private FilePath downloadDocker(DumbSlave slave, FilePath toolDir, String version) throws IOException, InterruptedException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); TeeOutputStream tee = new TeeOutputStream(baos, new PlainTextConsoleOutputStream(System.err)); TaskListener l = new StreamTaskListener(tee); FilePath exe = toolDir.child(version+"/bin/docker"); // Download for first time: assertEquals(exe.getRemote(), DockerTool.getExecutable(version, slave, l, null)); assertTrue(exe.exists()); assertThat(baos.toString(), containsString(Messages.DockerToolInstaller_downloading_docker_client_(version))); // Next time we do not need to download: baos.reset(); assertEquals(exe.getRemote(), DockerTool.getExecutable(version, slave, l, null)); assertTrue(exe.exists()); assertThat(baos.toString(), not(containsString(Messages.DockerToolInstaller_downloading_docker_client_(version)))); // Version check: baos.reset(); assertEquals(0, slave.createLauncher(l).launch().cmds(exe.getRemote(), "version", "--format", "{{.Client.Version}}").quiet(true).stdout(tee).stderr(System.err).join()); if (!version.equals("latest")) { assertEquals(version, baos.toString().trim()); } return exe; }
@Override public ServletOutputStream getOutputStream() throws IOException { return new ServletOutputStream() { private TeeOutputStream tee = new TeeOutputStream(ResponseWrapper.super.getOutputStream(), bos); @Override public boolean isReady() { return true; } @Override public void setWriteListener(WriteListener writeListener) { } @Override public void write(int b) throws IOException { tee.write(b); } }; }
public static void startPlatformLogging(Path fileName) { defaultSysOut = System.out; deafultSysErr = System.err; try { File file = fileName.toFile(); file.getParentFile().mkdirs(); file.createNewFile(); FileOutputStream fos = new FileOutputStream(file); TeeOutputStream bothStream = new TeeOutputStream(System.out, fos); PrintStream ps = new PrintStream(bothStream); System.setOut(ps); System.setErr(ps); } catch (Exception e) { e.printStackTrace(); LOG.error("Failed to redirect to log file %s", fileName); throw new GraphalyticsExecutionException("Failed to log the benchmark run. Benchmark run aborted."); } }
@Override public ServletOutputStream getOutputStream() throws IOException { return new ServletOutputStream() { private TeeOutputStream tee = new TeeOutputStream(ResponseWrapper.super.getOutputStream(), bos); @Override public void write(int b) throws IOException { tee.write(b); } @Override public boolean isReady() { // Auto-generated method stub return false; } @Override public void setWriteListener(WriteListener listener) { // Auto-generated method stub } }; }
public void setConsoleOutput(File local_output, boolean quiet, boolean silent) throws Exception { //Setting all system outputs to write to local_output file FileOutputStream logfile = new FileOutputStream(local_output); System.setOut(new PrintStream(new TeeOutputStream(System.out, logfile))); System.setOut(new PrintStream(new TeeOutputStream(System.err, logfile))); //Setting all LOGs to write to local_output file Properties prop = new Properties(); if (!quiet && !silent) { prop.setProperty("log4j.rootLogger", "INFO, console, WORKLOG"); prop.setProperty("log4j.appender.console","org.apache.log4j.ConsoleAppender"); prop.setProperty("log4j.appender.console.target", "System.err"); prop.setProperty("log4j.appender.console.layout", "org.apache.log4j.PatternLayout"); prop.setProperty("log4j.appender.console.layout.ConversionPattern", "%d [%p - %l] %m%n"); } else { prop.setProperty("log4j.rootLogger", "INFO, WORKLOG"); } prop.setProperty("log4j.appender.WORKLOG", "org.apache.log4j.FileAppender"); prop.setProperty("log4j.appender.WORKLOG.File", local_output.toString()); prop.setProperty("log4j.appender.WORKLOG.layout","org.apache.log4j.PatternLayout"); prop.setProperty("log4j.appender.WORKLOG.layout.ConversionPattern","%d %c{1} - %m%n"); PropertyConfigurator.configure(prop); }
private static OutputStream teeOutput(OutputStream capture, OutputStream user) { if (user == null) { return capture; } else { return new TeeOutputStream(capture, user); } }
/** * Runs the function runtime with the specified class and method * * @param cls class to thenRun * @param method the method name */ public void thenRun(String cls, String method) { InputStream oldSystemIn = System.in; PrintStream oldSystemOut = System.out; PrintStream oldSystemErr = System.err; try { PrintStream functionOut = new PrintStream(stdOut); PrintStream functionErr = new PrintStream(new TeeOutputStream(stdErr, oldSystemErr)); System.setOut(functionErr); System.setErr(functionErr); exitStatus = new EntryPoint().run( vars, pendingInput, functionOut, functionErr, cls + "::" + method); stdOut.flush(); stdErr.flush(); } catch (Exception e) { throw new RuntimeException(e); } finally { System.out.flush(); System.err.flush(); System.setIn(oldSystemIn); System.setOut(oldSystemOut); System.setErr(oldSystemErr); } }
private ExecuteStreamHandler createStreamHandler() throws IOException { out = new ByteArrayOutputStream(); err = new ByteArrayOutputStream(); PipedOutputStream outPiped = new PipedOutputStream(); InputStream inPiped = new PipedInputStream(outPiped); in = outPiped; TeeOutputStream teeOut = new TeeOutputStream(out, System.out); TeeOutputStream teeErr = new TeeOutputStream(err, System.err); return new PumpStreamHandler(teeOut, teeErr, inPiped); }
@Override byte[] writeContent(OutputStream out) throws IOException { ByteArrayOutputStream branch = new ByteArrayOutputStream(); TeeOutputStream tee = new TeeOutputStream(out, branch); writeInt(tag, 1, tee); writeInt(valuesCount, 1, tee); writeInt(bitmask, 1, tee); writeInt(controlByte, 1, tee); return branch.toByteArray(); }
@Override byte[] writeContent(OutputStream out) throws IOException { ByteArrayOutputStream branch = new ByteArrayOutputStream(); TeeOutputStream tee = new TeeOutputStream(out, branch); writeString(IDENTIFIER, 4, tee); writeInt(headerLength, 4, tee); writeInt(controlByteCount, 4, tee); for (MobiContentTagEntry tag : tags) { tag.writeContent(out); } return branch.toByteArray(); }
@Override byte[] writeContent(OutputStream out) throws IOException { ByteArrayOutputStream branch = new ByteArrayOutputStream(); TeeOutputStream tee = new TeeOutputStream(out, branch); writeString(IDENTIFIER, 4, tee); writeInt(headerLength, 4, tee); writeInt(indexType, 4, tee); writeInt(unknown1, 4, tee); writeInt(unknown2, 4, tee); writeInt(idxtIndex, 4, tee); writeInt(indexCount, 4, tee); writeInt(indexEncoding, 4, tee); writeInt(indexLanguage, 4, tee); writeInt(totalIndexCount, 4, tee); writeInt(ordtIndex, 4, tee); writeInt(ligtIndex, 4, tee); writeInt(ordtLigtEntriesCount, 4, tee); writeInt(cncxRecordCount, 4, tee); write(unknownIndxHeaderPart, out); if(tagx != null) { tagx.writeContent(tee); } write(rest, out); return branch.toByteArray(); }
@Override byte[] writeContent(OutputStream out) throws IOException { ByteArrayOutputStream branch = new ByteArrayOutputStream(); TeeOutputStream tee = new TeeOutputStream(out, branch); writeString(IDENTIFIER, 4, tee); for (int indexEntriesIndex : indexEntriesIndices) { writeInt(indexEntriesIndex, 2, tee); } writeInt(indexEntriesCount, 2, tee); return branch.toByteArray(); }
public LogSplitter() { try { this.stdoutPipeOutputStream = new PipedOutputStream(stdoutInput); this.stderrPipeOutputStream = new PipedOutputStream(stderrInput); this.combinedPipeOutputStream = new PipedOutputStream(combinedInput); this.stdoutOutput = new TeeOutputStream(stdoutPipeOutputStream, combinedPipeOutputStream); this.stderrOutput = new TeeOutputStream(stderrPipeOutputStream, combinedPipeOutputStream); } catch (IOException e) { throw new RuntimeException("Unable to create", e); } }
public GradleHandle start() { if (execHandle != null) { throw new IllegalStateException("you have already called start() on this handle"); } AbstractExecHandleBuilder execBuilder = execHandleFactory.create(); execBuilder.setStandardOutput(new CloseShieldOutputStream(new TeeOutputStream(System.out, standardOutput))); execBuilder.setErrorOutput(new CloseShieldOutputStream(new TeeOutputStream(System.err, errorOutput))); execHandle = execBuilder.build(); execHandle.start(); return this; }
/** * Constructor. */ public MockConsole() { this.prefixWithDate = false; this.outputBuffer = new ByteArrayOutputStream(); this.errorBuffer = new ByteArrayOutputStream(); OutputStream originalOut = this.out; this.out = new PrintStream( new TeeOutputStream( this.outputBuffer, this.out ) ); this.err = new PrintStream( new TeeOutputStream( this.errorBuffer, originalOut ) ); }
public static void main(final String[] args) throws IOException { final int topN = 20; final String baseDir = "/afs/inf.ed.ac.uk/user/j/jfowkes/Code/Sequences/"; // final String dataset = "jmlr"; // final String seqLabels = baseDir + "Datasets/JMLR/jmlr.lab"; final String dataset = "alice_punc"; final String seqLabels = baseDir + "Datasets/Alice/WithPunctuation/alice_punc.lab"; // Set up logging final FileOutputStream outFile = new FileOutputStream(baseDir + dataset + "_exclusive.txt"); final TeeOutputStream out = new TeeOutputStream(System.out, outFile); final PrintStream ps = new PrintStream(out); System.setOut(ps); final Map<Sequence, Double> ismSeqs = SequenceMiningCore .readISMSequences(new File(baseDir + "Logs/" + dataset + ".log")); final Map<Sequence, Double> sqsSeqs = StatisticalSequenceMining .readSQSSequences(new File(baseDir + "SQS/" + dataset + ".txt")); final Map<Sequence, Double> gokrimpSeqs = StatisticalSequenceMining .readGoKrimpSequences(new File(baseDir + "GoKrimp/" + dataset + ".txt")); final Set<Sequence> ISMnotSQSorGoKrimp = getExclusiveSequences(ismSeqs.keySet(), sqsSeqs.keySet(), gokrimpSeqs.keySet()); final Set<Sequence> SQSnotISMorGoKrimp = getExclusiveSequences(sqsSeqs.keySet(), ismSeqs.keySet(), gokrimpSeqs.keySet()); final Set<Sequence> GoKrimpnotISMorSQS = getExclusiveSequences(gokrimpSeqs.keySet(), ismSeqs.keySet(), sqsSeqs.keySet()); final List<String> dict = FileUtils.readLines(new File(seqLabels)); // Print top ten System.out.print("\n============= ISM not SQS/GoKrimp =============\n"); printTopExclusiveSequences(topN, ismSeqs, ISMnotSQSorGoKrimp, dict); System.out.print("\n============= SQS not ISM/GoKrimp =============\n"); printTopExclusiveSequences(topN, sqsSeqs, SQSnotISMorGoKrimp, dict); System.out.print("\n============= GoKrimp not ISM/SQS =============\n"); printTopExclusiveSequences(topN, gokrimpSeqs, GoKrimpnotISMorSQS, dict); }
public static void main(final String[] args) throws IOException, ClassNotFoundException { final String baseFolder = "/afs/inf.ed.ac.uk/user/j/jfowkes/Code/Sequences/"; // final File dbFile = new File(baseFolder + "Datasets/parallel", // ".dat"); // generateParallelDataset(dbFile); // Set up logging final FileOutputStream outFile = new FileOutputStream(baseFolder + "PrecisionRecall/parallel_pr.txt"); final TeeOutputStream out = new TeeOutputStream(System.out, outFile); final PrintStream ps = new PrintStream(out); System.setOut(ps); // Read SQS sequences final File outSQS = new File(baseFolder + "SQS/parallel_partial.txt"); final Map<Sequence, Double> seqsSQS = StatisticalSequenceMining.readSQSSequences(outSQS); // Read GoKrimp sequences final File outGOKRIMP = new File(baseFolder + "GoKrimp/parallel.txt"); final Map<Sequence, Double> seqsGORKIMP = StatisticalSequenceMining.readGoKrimpSequences(outGOKRIMP); // Read ISM sequences final File outISM = new File(baseFolder + "Logs/parallel.log"); final Map<Sequence, Double> seqsISM = SequenceMining.readISMSequences(outISM); // Precision-recall precisionRecall(seqsSQS, "SQS"); precisionRecall(seqsGORKIMP, "GoKrimp"); precisionRecall(seqsISM, "ISM"); }
public String runTest(String[] args) throws Exception { final PrintStream stdout = System.out; final ByteArrayOutputStream myOut = new ByteArrayOutputStream(); TeeOutputStream tee = new TeeOutputStream(stdout, myOut); System.setOut(new PrintStream(tee)); final long t0 = System.nanoTime(); Main.main(args); final String standardOutput = myOut.toString(); System.setOut(stdout); System.out.println((System.nanoTime() - t0) / 1000000000 + "s"); return standardOutput; }
private ProcReturn execCommand(boolean quiet, String... cmd) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); Launcher launcher = decorator .decorate(new DummyLauncher(new StreamTaskListener(new TeeOutputStream(out, System.out))), null); ContainerExecProc proc = (ContainerExecProc) launcher .launch(launcher.new ProcStarter().pwd("/tmp").cmds(cmd).quiet(quiet)); // wait for proc to finish (shouldn't take long) while (proc.isAlive()) { Thread.sleep(100); } assertFalse("proc is alive", proc.isAlive()); int exitCode = proc.joinWithTimeout(10, TimeUnit.SECONDS, StreamTaskListener.fromStderr()); return new ProcReturn(proc, exitCode, out.toString()); }
@Around("call(void org.apache.axis2.transport.MessageFormatter.writeTo(" + " org.apache.axis2.context.MessageContext, org.apache.axiom.om.OMOutputFormat," + " java.io.OutputStream, boolean))" + " && args(msgContext, format, out, preserve)") public void aroundWriteTo(ProceedingJoinPoint proceedingJoinPoint, MessageContext msgContext, OMOutputFormat format, OutputStream out, boolean preserve) throws Throwable { OutputStream log = LogManager.INSTANCE.createLog("formatter"); try { OutputStream tee = new TeeOutputStream(out, log); proceedingJoinPoint.proceed(new Object[] { msgContext, format, tee, preserve }); } finally { log.close(); } }
private Channel openChannel(Session session, String channelType) throws JSchException { Channel channel = session.openChannel(channelType); channel.setOutputStream(new TeeOutputStream(bout, System.out)); channel.connect(); return channel; }
/** Initializes logging facility, called once by VisEditor. */ public static void init () { if (initialized) throw new IllegalStateException("Log cannot be initialized twice!"); initialized = true; prepareLogFile(); System.setOut(new PrintStream(new TeeOutputStream(System.out, logFileStream))); System.setErr(new PrintStream(new TeeOutputStream(System.err, logFileStream))); }
@Before public final void captureStdout() { originalStdout = System.out; originalStderr = System.err; capturedOutputBytes = new ByteArrayOutputStream(); System.setOut(new PrintStream(new TeeOutputStream(originalStdout, capturedOutputBytes))); System.setErr(new PrintStream(new TeeOutputStream(originalStderr, capturedOutputBytes))); }
public OutputCapturer(OutputStream standardStream, String outputEncoding) { this.buffer = new ByteArrayOutputStream(); this.outputStream = new CloseShieldOutputStream(new TeeOutputStream(standardStream, buffer)); this.outputEncoding = outputEncoding; }
public TeeServletOutputStream(OutputStream one, OutputStream two) { targetStream = new TeeOutputStream(one, two); }
/** * Runs the function runtime with the specified class and method (and waits for Flow stages to finish * if the test spawns any Flow) * * @param cls class to thenRun * @param method the method name */ public void thenRun(String cls, String method) { final ClassLoader functionClassLoader; Class c = null; try { // Trick to work around Maven class loader separation // if passed class is a valid class then set the classloader to the same as the class's loader c = Class.forName(cls); } catch (Exception ignored) { // TODO don't fall through here } if (c != null) { functionClassLoader = c.getClassLoader(); } else { functionClassLoader = getClass().getClassLoader(); } PrintStream oldSystemOut = System.out; PrintStream oldSystemErr = System.err; InMemCompleter.CompleterInvokeClient client = new TestRuleCompleterInvokeClient(functionClassLoader, oldSystemErr, cls, method); InMemCompleter.FnInvokeClient fnInvokeClient = new TestRuleFnInvokeClient(); // FlowContinuationInvoker.setTestingMode(true); // The following must be a static: otherwise the factory (the lambda) will not be serializable. completer = new InMemCompleter(client, fnInvokeClient); //TestSupport.installCompleterClientFactory(completer, oldSystemErr); Map<String, String> mutableEnv = new HashMap<>(); try { PrintStream functionOut = new PrintStream(stdOut); PrintStream functionErr = new PrintStream(new TeeOutputStream(stdErr, oldSystemErr)); System.setOut(functionErr); System.setErr(functionErr); mutableEnv.putAll(config); mutableEnv.putAll(eventEnv); mutableEnv.put("FN_FORMAT", "http"); FnTestingClassLoader forked = new FnTestingClassLoader(functionClassLoader, sharedPrefixes); if (forked.isShared(cls)) { oldSystemErr.println("WARNING: The function class under test is shared with the test ClassLoader."); oldSystemErr.println(" This may result in unexpected behaviour around function initialization and configuration."); } forked.setCompleterClient(completer); lastExitCode = forked.run( mutableEnv, pendingInput, functionOut, functionErr, cls + "::" + method); stdOut.flush(); stdErr.flush(); completer.awaitTermination(); } catch (Exception e) { throw new RuntimeException("internal error raised by entry point or flushing the test streams", e); } finally { System.out.flush(); System.err.flush(); System.setOut(oldSystemOut); System.setErr(oldSystemErr); } }
@Nonnull public static StdStreams tee(@Nonnull StdStreams a, @Nonnull StdStreams b) { final OutputStream out = new TeeOutputStream(a.out(), b.out()); final OutputStream err = new TeeOutputStream(a.err(), b.err()); return new Impl(out, err); }
@Override byte[] writeContent(OutputStream out) throws IOException { ByteArrayOutputStream branch = new ByteArrayOutputStream(); TeeOutputStream tee = new TeeOutputStream(out, branch); writeInt(compression, 2, tee); writeInt(unused0, 2, tee); writeInt(textLength, 4, tee); writeInt(recordCount, 2, tee); writeInt(recordSize, 2, tee); writeInt(encryptionType, 2, tee); writeInt(unused1, 2, tee); writeString("MOBI", 4, tee); writeInt(DEFAULT_HEADER_LENGTH, 4, tee); writeInt(mobiType, 4, tee); writeInt(textEncoding, 4, tee); writeInt(uniqueID, 4, tee); writeInt(fileVersion, 4, tee); writeInt(orthographicIndex, 4, tee); writeInt(inflectionIndex, 4, tee); writeInt(indexNames, 4, tee); writeInt(indexKeys, 4, tee); writeInt(extraIndex0, 4, tee); writeInt(extraIndex1, 4, tee); writeInt(extraIndex2, 4, tee); writeInt(extraIndex3, 4, tee); writeInt(extraIndex4, 4, tee); writeInt(extraIndex5, 4, tee); writeInt(firstNonBookIndex, 4, tee); writeInt(fullNameOffset, 4, tee); writeInt(fullNameLength, 4, tee); writeInt(locale, 4, tee); writeInt(inputLanguage, 4, tee); writeInt(outputLanguage, 4, tee); writeInt(minVersion, 4, tee); writeInt(firstImageIndex, 4, tee); writeInt(huffmanRecordOffset, 4, tee); writeInt(huffmanRecordCount, 4, tee); writeInt(huffmanTableOffset, 4, tee); writeInt(huffmanTableLength, 4, tee); writeInt(exthFlags, 4, tee); // optional header part with 148 bytes write(new byte[32], tee); // Unknown 32 bytes writeInt(-1, 4, tee); // Unknown Use 0xFFFFFFFF writeInt(-1, 4, tee); // DRM-Offset: No DRM writeInt(0, 4, tee); // DRM-Count: No DRM writeInt(0, 4, tee); // DRM-Size: No DRM writeInt(0, 4, tee); // DRM-Flags: No DRM write(new byte[8], tee); // Unknown writeInt(firstContentRecordIndex, 2, tee); writeInt(lastContentRecordIndex, 2, tee); writeInt(1, 4, tee); // Unknown 0x00000001 writeInt(fcisRecordIndex, 4, tee); writeInt(fcisRecordCount, 4, tee); writeInt(flisRecordIndex, 4, tee); writeInt(flisRecordCount, 4, tee); write(new byte[8], tee); // Unknown 8 bytes writeInt(srcsRecordIndex, 4, tee); writeInt(srcsRecordCount, 4, tee); writeInt(-1, 4, tee); // Unknown Use 0xFFFFFFFF writeInt(-1, 4, tee); // Unknown Use 0xFFFFFFFF writeInt(extraRecordDataFlags, 4, tee); writeInt(indxRecordIndex, 4, tee); writeInt(-1, 4, tee); // unknown writeInt(fragmentRecordIndex, 4, tee); writeInt(-1, 4, tee); // unknown writeInt(skeletonRecordIndex, 4, tee); writeInt(datpRecordIndex, 4, tee); writeInt(0, 4, tee); // Unknown writeInt(guideIndex, 4, tee); // Unknown writeInt(0, 4, tee); // Unknown if(exthExists()) { exthHeader.writeEXTHHeader(tee); } write(remainder, remainder.length, tee); return branch.toByteArray(); }
public TeeServletOutputStream( OutputStream one, OutputStream two ) { targetStream = new TeeOutputStream( one, two); }
public static void main(final String[] args) throws IOException { // TODO re-run BIDE... final int topN = 50; final String baseDir = "/afs/inf.ed.ac.uk/user/j/jfowkes/Code/Sequences/"; final String[] datasets = new String[] { "alice_punc", "GAZELLE1", "jmlr", "SIGN", "aslbu", "aslgt", "auslan2", "context", "pioneer", "skating" }; // Set up logging final FileOutputStream outFile = new FileOutputStream(baseDir + "redundancy.txt"); final TeeOutputStream out = new TeeOutputStream(System.out, outFile); final PrintStream ps = new PrintStream(out); System.setOut(ps); final Writer writer = Files.newWriter(new File(baseDir + "redundancy.tex"), Charsets.UTF_8); for (int i = 0; i < datasets.length; i++) { System.out.println("===== Dataset: " + datasets[i]); // ISM sequences final Map<Sequence, Double> intSequences = SequenceMiningCore .readISMSequences(new File(baseDir + "Logs/" + datasets[i] + ".log")); calculateRedundancyStats("ISM", intSequences, topN, writer); // SQS sequences final Map<Sequence, Double> sqsSequences = StatisticalSequenceMining .readSQSSequences(new File(baseDir + "SQS/" + datasets[i] + ".txt")); calculateRedundancyStats("SQS", sqsSequences, topN, writer); // GoKrimp sequences final Map<Sequence, Double> gokrimpSequences = StatisticalSequenceMining .readGoKrimpSequences(new File(baseDir + "GoKrimp/" + datasets[i] + ".txt")); calculateRedundancyStats("GoKrimp", gokrimpSequences, topN, writer); // BIDE sequences final Map<Sequence, Integer> bideSequences = FrequentSequenceMining .readFrequentSequences(new File(baseDir + "BIDE/" + datasets[i] + ".txt")); calculateRedundancyStats("BIDE", bideSequences, topN, writer); System.out.println(); } writer.close(); }
public static void scalingTransactions(final int noCores, final int[] trans) throws IOException, ClassNotFoundException { final double[] time = new double[trans.length]; final DecimalFormat formatter = new DecimalFormat("0.0E0"); // Save to file final FileOutputStream outFile = new FileOutputStream(saveDir + "/" + name + "_scaling_" + noCores + ".txt"); final TeeOutputStream out = new TeeOutputStream(System.out, outFile); final PrintStream ps = new PrintStream(out); System.setOut(ps); // Read in previously mined sequences final Map<Sequence, Double> sequences = SequenceMiningCore.readISMSequences(sequenceLog); System.out.print("\n============= ACTUAL SEQUENCES =============\n"); for (final Entry<Sequence, Double> entry : sequences.entrySet()) { System.out.print(String.format("%s\tprob: %1.5f %n", entry.getKey(), entry.getValue())); } System.out.println("\nNo sequences: " + sequences.size()); System.out.println("No items: " + countNoItems(sequences.keySet())); // Read in associated sequence count distribution @SuppressWarnings("unchecked") final Table<Sequence, Integer, Double> countDist = (Table<Sequence, Integer, Double>) Logging .deserializeFrom(FilenameUtils.removeExtension(sequenceLog.getAbsolutePath()) + ".dist"); transloop: for (int i = 0; i < trans.length; i++) { final int tran = trans[i]; System.out.println("\n========= " + formatter.format(tran) + " Transactions"); // Generate transaction database TransactionGenerator.generateTransactionDatabase(sequences, countDist, tran, dbFile); SequenceScaling.printTransactionDBStats(dbFile); // Mine sequences final File logFile = Logging.getLogFileName("ISM", true, saveDir, dbFile); final long startTime = System.currentTimeMillis(); SequenceMining.mineSequences(dbFile, new InferGreedy(), maxStructureSteps, maxEMIterations, logFile, false); final long endTime = System.currentTimeMillis(); final double tim = (endTime - startTime) / (double) 1000; time[i] += tim; System.out.printf("Time (s): %.2f%n", tim); if (tim > MAX_RUNTIME * 60) break transloop; } // Print time System.out.println("\n========" + name + "========"); System.out.println("Transactions:" + Arrays.toString(trans)); System.out.println("Time: " + Arrays.toString(time)); // and save to file out.close(); }
public static void precisionRecall(final HashMap<Itemset, Double> itemsets, final HashMap<Itemset, Double> specialItemsets, final String algorithm) throws IOException { // Set up logging final FileOutputStream outFile = new FileOutputStream(saveDir + "/" + algorithm + "_" + name + "_pr.txt"); final TeeOutputStream out = new TeeOutputStream(System.out, outFile); final PrintStream ps = new PrintStream(out); System.setOut(ps); System.out.println("\nSpecial Itemsets: " + noSpecialItemsets); // Mine itemsets Map<Itemset, Double> minedItemsets = null; final File logFile = Logging.getLogFileName(algorithm, true, saveDir, dbFile); final long startTime = System.currentTimeMillis(); if (algorithm.equals("spark")) minedItemsets = runSpark(sparkCores, noIterations); else if (algorithm.equals("MTV")) minedItemsets = StatisticalItemsetMining.mineMTVItemsets(dbFile, minSup, noMTVIterations, logFile); else if (algorithm.equals("CHARM")) { CondensedFrequentItemsetMining.mineClosedFrequentItemsetsCharm(dbFile.getAbsolutePath(), logFile.getAbsolutePath(), minSup); minedItemsets = FrequentItemsetMining.readFrequentItemsetsChiSquared(logFile, dbFile.getAbsolutePath()); } else if (algorithm.equals("IIM")) minedItemsets = ItemsetMining.mineItemsets(dbFile, new InferGreedy(), maxStructureSteps, noIterations, logFile); else if (algorithm.equals("KRIMP")) minedItemsets = StatisticalItemsetMining.mineKRIMPItemsets(dbFile, (int) Math.floor(minSup * noTransactions)); else if (algorithm.equals("SLIM")) minedItemsets = StatisticalItemsetMining.mineSLIMItemsets(dbFile, (int) Math.floor(minSup * noTransactions), 24); else if (algorithm.equals("Tiling")) minedItemsets = StatisticalItemsetMining.mineTilingItemsets(dbFile, minSup); else throw new RuntimeException("Incorrect algorithm name."); final long endTime = System.currentTimeMillis(); final double time = (endTime - startTime) / (double) 1000; // Calculate sorted precision and recall final int len = minedItemsets.size(); System.out.println("No. mined itemsets: " + len); final double[] precision = new double[len]; final double[] recall = new double[len]; for (int k = 1; k <= len; k++) { final Set<Itemset> topKMined = Sets.newHashSet(); for (final Entry<Itemset, Double> entry : minedItemsets.entrySet()) { topKMined.add(entry.getKey()); if (topKMined.size() == k) break; } final double noInBoth = Sets.intersection(itemsets.keySet(), topKMined).size(); final double noSpecialInBoth = Sets.intersection(specialItemsets.keySet(), topKMined).size(); final double pr = noInBoth / (double) topKMined.size(); final double rec = noSpecialInBoth / (double) specialItemsets.size(); precision[k - 1] = pr; recall[k - 1] = rec; } // Output precision and recall System.out.println("\n======== " + name + " ========"); System.out.println("Special Frequency: " + noSpecialItemsets); System.out.println("Time: " + time); System.out.println("Precision (all): " + Arrays.toString(precision)); System.out.println("Recall (special): " + Arrays.toString(recall)); }
public static void precisionRecall(final HashMap<Itemset, Double> itemsets, final String algorithm) throws IOException { // Set up logging final FileOutputStream outFile = new FileOutputStream(saveDir + "/" + algorithm + "_" + name + "_pr.txt"); final TeeOutputStream out = new TeeOutputStream(System.out, outFile); final PrintStream ps = new PrintStream(out); System.setOut(ps); // Mine itemsets Map<Itemset, Double> minedItemsets = null; final File logFile = Logging.getLogFileName(algorithm, true, saveDir, dbFile); final long startTime = System.currentTimeMillis(); if (algorithm.equals("spark")) minedItemsets = ItemsetPrecisionRecall.runSpark(sparkCores, noIterations); else if (algorithm.equals("MTV")) minedItemsets = StatisticalItemsetMining.mineMTVItemsets(dbFile, minSup, noMTVIterations, logFile); else if (algorithm.equals("CHARM")) { CondensedFrequentItemsetMining.mineClosedFrequentItemsetsCharm(dbFile.getAbsolutePath(), logFile.getAbsolutePath(), minSup); minedItemsets = FrequentItemsetMining.readFrequentItemsetsChiSquared(logFile, dbFile.getAbsolutePath()); } else if (algorithm.equals("IIM")) minedItemsets = ItemsetMining.mineItemsets(dbFile, new InferGreedy(), maxStructureSteps, noIterations, logFile); else if (algorithm.equals("KRIMP")) minedItemsets = StatisticalItemsetMining.mineKRIMPItemsets(dbFile, (int) Math.floor(minSup * noTransactions)); else if (algorithm.equals("SLIM")) minedItemsets = StatisticalItemsetMining.mineSLIMItemsets(dbFile, (int) Math.floor(minSup * noTransactions), 24); else if (algorithm.equals("Tiling")) minedItemsets = StatisticalItemsetMining.mineTilingItemsets(dbFile, minSup); else throw new RuntimeException("Incorrect algorithm name."); final long endTime = System.currentTimeMillis(); final double time = (endTime - startTime) / (double) 1000; // Calculate sorted precision and recall final int len = minedItemsets.size(); System.out.println("No. mined itemsets: " + len); final double[] precision = new double[len]; final double[] recall = new double[len]; for (int k = 1; k <= minedItemsets.size(); k++) { final Set<Itemset> topKMined = Sets.newHashSet(); for (final Entry<Itemset, Double> entry : minedItemsets.entrySet()) { topKMined.add(entry.getKey()); if (topKMined.size() == k) break; } final double noInBoth = Sets.intersection(itemsets.keySet(), topKMined).size(); final double pr = noInBoth / (double) topKMined.size(); final double rec = noInBoth / (double) itemsets.size(); precision[k - 1] = pr; recall[k - 1] = rec; } // Output precision and recall System.out.println("\n======== " + name + " ========"); System.out.println("Time: " + time); System.out.println("Precision (all): " + Arrays.toString(precision)); System.out.println("Recall (all): " + Arrays.toString(recall)); }
public static void scalingTransactions(final int noCores, final int[] trans) throws IOException, InterruptedException { final double[] time = new double[trans.length]; final DecimalFormat formatter = new DecimalFormat("0.0E0"); // Save to file String prefix = ""; if (useSpark) prefix = "spark_"; final FileOutputStream outFile = new FileOutputStream(saveDir + "/" + prefix + name + "_scaling_" + noCores + ".txt"); final TeeOutputStream out = new TeeOutputStream(System.out, outFile); final PrintStream ps = new PrintStream(out); System.setOut(ps); // Read in previously mined itemsets final Map<Itemset, Double> itemsets = ItemsetMiningCore .readIIMItemsets(itemsetLog); System.out.print("\n============= ACTUAL ITEMSETS =============\n"); for (final Entry<Itemset, Double> entry : itemsets.entrySet()) { System.out.print(String.format("%s\tprob: %1.5f %n", entry.getKey(), entry.getValue())); } System.out.println("\nNo itemsets: " + itemsets.size()); System.out.println("No items: " + countNoItems(itemsets.keySet())); transloop: for (int i = 0; i < trans.length; i++) { final int tran = trans[i]; System.out.println("\n========= " + formatter.format(tran) + " Transactions"); // Generate transaction database TransactionGenerator.generateTransactionDatabase(itemsets, tran, dbFile); printTransactionDBStats(dbFile); // Mine itemsets final File logFile = Logging.getLogFileName("IIM", true, saveDir, dbFile); final long startTime = System.currentTimeMillis(); if (useSpark) runSpark(noCores); else ItemsetMining.mineItemsets(dbFile, new InferGreedy(), maxStructureSteps, maxEMIterations, logFile); final long endTime = System.currentTimeMillis(); final double tim = (endTime - startTime) / (double) 1000; time[i] += tim; System.out.printf("Time (s): %.2f%n", tim); if (tim > MAX_RUNTIME * 60) break transloop; } // Print time System.out.println("\n========" + name + "========"); System.out.println("Transactions:" + Arrays.toString(trans)); System.out.println("Time: " + Arrays.toString(time)); // and save to file out.close(); }