/** * Retrieves completed assignments for a HIT. You can get assignments for a HIT at any time, * even if the HIT is not yet "reviewable". If a HIT has multiple assignments, and has * received some results but has not yet become "reviewable", you can still retrieve the partial * results with this method. * * The results are sorted by SubmitTime. * * @param hitId the Id of the HIT for which completed assignments are to be returned * @param pageNum The page of results to return. Once the assignments have been filtered, * sorted, and divided into pages, the page corresponding * to pageNum is returned as the results of the operation. * @param getFullResponse if true, all properties of the HIT are returned. If false, only the HIT Id * and HIT type Id are returned. * @return an array of Assignments * @throws ServiceException */ public Assignment[] getAssignmentsForHIT(String hitId, int pageNum, boolean getFullResponse) throws ServiceException { // Include AssignmentFeedback in response String[] responseGroup = null; if (getFullResponse == true) { responseGroup = DEFAULT_ASSIGNMENT_RESPONSE_GROUP; } GetAssignmentsForHITResult result = super.getAssignmentsForHIT(hitId, DEFAULT_SORT_DIRECTION, DEFAULT_ASSIGNMENT_STATUS, GetAssignmentsForHITSortProperty.SubmitTime, pageNum, DEFAULT_PAGE_SIZE, responseGroup ); return result.getAssignment(); }
private List<Assignment> getAllAssignmentsForHIT( HIT hit) { String name = "getAllAssignmentsForHIT"; int waittime = 2; while (true) { synchronized (service) { try { Assignment[] hitAssignments = service.getAllAssignmentsForHIT(hit.getHITId()); List<Assignment> assignments = new LinkedList<>(); boolean addAll = assignments.addAll(Arrays.asList(hitAssignments)); if (addAll) LOGGER.info(String.format("Retrieved %d assignments for HIT %s", hitAssignments.length, hit.getHITId())); return assignments; } catch (InternalServiceException ise) { LOGGER.warn(format("{0} {1}", name, ise)); chill(waittime); waittime *= 2; } } } }
public static org.cmuchimps.gort.modules.dataobject.Assignment createDBAssignment(GortEntityManager gem, EntityManager em, Assignment mturkAssignment) { if (gem == null || em == null || mturkAssignment == null) { return null; } org.cmuchimps.gort.modules.dataobject.Assignment retVal = new org.cmuchimps.gort.modules.dataobject.Assignment(); retVal.setAnswer(mturkAssignment.getAnswer()); retVal.setAssignmentId(mturkAssignment.getAssignmentId()); retVal.setWorkerId(mturkAssignment.getWorkerId()); retVal.setAssignmentStatus(mturkAssignment.getAssignmentStatus().getValue()); retVal.setSubmission(mturkAssignment.getSubmitTime().getTime()); gem.insertEntity(em, retVal); return retVal; }
@SuppressWarnings("unchecked") private String getAnswers(Assignment assignment) { String result = EMPTY; AssignmentStatus status = assignment.getAssignmentStatus(); if (status == null) { return NO_ANSWER; } String answerXML = assignment.getAnswer(); QuestionFormAnswers qfa = RequesterService.parseAnswers(answerXML); List<QuestionFormAnswersType.AnswerType> answers = (List<QuestionFormAnswersType.AnswerType>) qfa.getAnswer(); for (QuestionFormAnswersType.AnswerType answer : answers) { String assignmentId = assignment.getAssignmentId(); String answerValue = RequesterService.getAnswerValue(assignmentId, answer, true); if (answerValue != null) { result += answerValue + DELIMITER; } } return result; }
private boolean addResults(GetAssignmentsForHITResult currentResults) { Assignment[] assignments = currentResults.getAssignment(); if (assignments != null) { // Add the results Collections.addAll(allResults, assignments); // if there are more results, load them if (assignments.length == pageSize && currentResults.getTotalNumResults() > pageSize) { return true; } } return false; }
/**] * Returns the assignments for the HIT to the caller */ public Assignment[] getResults() throws Exception { // make sure worker thread is done loading assignments firstReply.getResult(); if (exception != null) { throw exception; } return (Assignment[]) allResults.toArray(new Assignment[allResults.size()]); }
/** * Retrieves all assignments that have the * the specified status. * * @param hitId the Id of the HIT for which the assignments are returned * @param status the status value of the assignments to return. * @return an array of Assignments * @throws ServiceException */ public Assignment[] getAllAssignmentsForHIT(String hitId, AssignmentStatus[] status) throws ServiceException { List<Assignment> results = new ArrayList<Assignment>(); int pageNum = 1; do { GetAssignmentsForHITResult result = super.getAssignmentsForHIT(hitId, DEFAULT_SORT_DIRECTION, status, GetAssignmentsForHITSortProperty.SubmitTime, pageNum, DEFAULT_PAGE_SIZE, DEFAULT_ASSIGNMENT_RESPONSE_GROUP ); Assignment[] assignment = result.getAssignment(); if (assignment != null) { // Add the results Collections.addAll(results, assignment); } // Check if we're on the last page or not if (assignment == null || assignment.length < DEFAULT_PAGE_SIZE) break; pageNum++; } while (true); return (Assignment[]) results.toArray(new Assignment[results.size()]); }
public static Assignment[] getSubmittedAssignments(RequesterService service, String hitId) { if (service == null) { return null; } if (hitId == null || hitId.isEmpty()) { return null; } Assignment[] assignments = null; try { assignments = service.getAllAssignmentsForHIT(hitId); } catch (Exception e) { System.out.println("Could not connect to receive HIT assignments."); //e.printStackTrace(); } if (assignments == null || assignments.length <= 0) { return null; } List<Assignment> retVal = new LinkedList<Assignment>(); //Only assignments that have been submitted will contain answer data for (Assignment a : assignments) { if (a == null) { continue; } if (a.getAssignmentStatus() == AssignmentStatus.Submitted) { retVal.add(a); } } return (retVal.size() > 0) ? retVal.toArray(new Assignment[retVal.size()]) : null; }
private Date getHITAssignmentsMaxDate(GortEntityManager gem, EntityManager em, HIT hit) { // refresh the hit to get all its assignments em.refresh(hit); List<org.cmuchimps.gort.modules.dataobject.Assignment> assignments = hit.getAssignments(); if (assignments == null) { return null; } Iterator<org.cmuchimps.gort.modules.dataobject.Assignment> iterator = assignments.iterator(); Date max = null; while (iterator.hasNext()) { org.cmuchimps.gort.modules.dataobject.Assignment assignment = iterator.next(); Date submission = assignment.getSubmission(); if (submission == null) { continue; } if (max == null) { max = submission; continue; } else if (max.before(submission)) { max = submission; } } return max; }
public HITResults(HIT hit, Assignment[] assignments, ClientConfig config) { this.hit = hit; this.assignments = assignments; this.config = config; }
public Assignment[] getAssignments() { return this.assignments; }
public void writeResults(HITDataOutput writer) throws IOException { Map<String,String> hitResults = this.getHITResults(); Map<String,String> assignmentResults; if (assignments == null || assignments.length == 0) { assignments = NO_ASSIGNMENTS; // Feature 1826272 (output empty assignment data even when HIT has no assignments) } for (Assignment assignment : this.assignments) { assignmentResults = this.getAssignmentResults(assignment); assignmentResults.putAll( hitResults ); writer.writeValues( assignmentResults ); // append } }
public Map<String,String> getAssignmentResults(Assignment assignment) { // Add standard Assignment results Map<String,String> results = new LinkedHashMap<String,String>( HITTypeResults.ASSIGNMENT_HEADERS.length ); results.put( HITProperties.AssignmentField.AssignmentId.getFieldName(), assignment.getAssignmentId()); results.put( HITProperties.AssignmentField.WorkerId.getFieldName(), assignment.getWorkerId()); AssignmentStatus status = assignment.getAssignmentStatus(); String statusStr = status != null ? status.getValue() : EMPTY; results.put( HITProperties.AssignmentField.Status.getFieldName(), statusStr); String autoApprovalTime = assignment.getAutoApprovalTime() != null ? DATE_FORMATTER.format(assignment.getAutoApprovalTime().getTime()) : EMPTY; results.put( HITProperties.AssignmentField.AutoApprovalTime.getFieldName(), autoApprovalTime); String acceptTime = assignment.getAcceptTime() != null ? DATE_FORMATTER.format(assignment.getAcceptTime().getTime()) : EMPTY; results.put( HITProperties.AssignmentField.AcceptTime.getFieldName(), acceptTime); String submitTime = assignment.getSubmitTime() != null ? DATE_FORMATTER.format(assignment.getSubmitTime().getTime()) : EMPTY; results.put( HITProperties.AssignmentField.SubmitTime.getFieldName(), submitTime); String approvalTime = assignment.getApprovalTime() != null ? DATE_FORMATTER.format(assignment.getApprovalTime().getTime()) : EMPTY; results.put( HITProperties.AssignmentField.ApprovalTime.getFieldName(), approvalTime); String rejectionTime = assignment.getRejectionTime() != null ? DATE_FORMATTER.format(assignment.getRejectionTime().getTime()) : EMPTY; results.put( HITProperties.AssignmentField.RejectionTime.getFieldName(), rejectionTime); String deadline = assignment.getDeadline() != null ? DATE_FORMATTER.format(assignment.getDeadline().getTime()) : EMPTY; results.put( HITProperties.AssignmentField.Deadline.getFieldName(), deadline); String requesterFeedback = assignment.getRequesterFeedback() != null ? assignment.getRequesterFeedback() : EMPTY; results.put( HITProperties.AssignmentField.RequesterFeedback.getFieldName(), requesterFeedback); String rejectFlag = status != null && status == AssignmentStatus.Rejected ? "y" : EMPTY; results.put( HITProperties.AssignmentField.RejectFlag.getFieldName(), rejectFlag); // Add Assignment-specific answers String answers = this.getAnswers(assignment); results.put( HITProperties.AssignmentField.Answers.getFieldName(), answers); return results; }
/** * Loads all assignment pages for a HIT using the Axis worker thread pool. */ public Assignment[] getAssignmentsForHITAsync(String hitId, SortDirection sortDirection, AssignmentStatus[] status, GetAssignmentsForHITSortProperty sortProperty, Integer pageSize, String[] responseGroup, GetAssignmentsForHITResult firstPage, AsyncCallback callback) throws ServiceException { Assignment[] ret = new Assignment[] {}; GetAssignmentsForHITResult result=null; if (firstPage==null) { // get first page to find how many assignments there are AsyncReply first = getAssignmentsForHITAsync(hitId, sortDirection, status, sortProperty, 1, pageSize, responseGroup, callback); result = ((GetAssignmentsForHITResult[])first.getResult())[0]; if (result.getAssignment() != null) { ret = result.getAssignment(); } } else { result = firstPage; } // check size and total size and create subsequent requests if necessary if (ret.length == pageSize && result.getTotalNumResults() > pageSize) { // there are more results List<Assignment> results = new ArrayList<Assignment>(); Collections.addAll(results, ret); int numPages = result.getTotalNumResults()/pageSize; AsyncReply[] replies = new AsyncReply[numPages]; for (int i=0; i<numPages; i++) { replies[i] = getAssignmentsForHITAsync(hitId, sortDirection, status, sortProperty, i+1, pageSize, responseGroup, callback); } // append results for (int i=0; i<numPages; i++) { result = ((GetAssignmentsForHITResult[])replies[i].getResult())[0]; if (result.getAssignment() != null) { Collections.addAll(results, result.getAssignment()); } } ret = (Assignment[]) results.toArray(new Assignment[results.size()]); } return ret; }
/** * Fetches and prints a bunch of review policy info about the given HIT. * @param hitId Which HIT to print results for */ public void printReviewResults(String hitId) { service.getRequesterStatistic(RequesterStatistic.EstimatedTotalLiability, TimePeriod.LifeToDate, null); System.out.println("Getting review policy results for HIT " + hitId + "..."); System.out.println(); GetReviewResultsForHITResult results = service.getReviewResultsForHIT( hitId, new ReviewPolicyLevel[] {ReviewPolicyLevel.Assignment, ReviewPolicyLevel.HIT}, true, // retrieveActions true, // retrieveResults 1, // pageNumber 1000, // pageSize null); // responseGroup // Print ScoreKnownAnswers results/actions for this HIT printReviewReport(results.getAssignmentReviewReport(), results.getAssignmentReviewPolicy().getPolicyName()); // Print PluralityHitReview results/actions for this HIT printReviewReport(results.getHITReviewReport(), results.getHITReviewPolicy().getPolicyName()); // Get a list of workers who worked on this HIT Assignment[] assignments = service.getAssignmentsForHIT(hitId, 1); Set<String> workerIds = new HashSet<String>(); if (assignments != null) { for (Assignment assignment : assignments) { workerIds.add(assignment.getWorkerId()); } } // For each of those workers, fetch statistics relevant to review policies RequesterStatistic[] statistics = { // which statistics to print RequesterStatistic.NumberKnownAnswersCorrect, RequesterStatistic.NumberKnownAnswersIncorrect, RequesterStatistic.NumberKnownAnswersEvaluated, RequesterStatistic.PercentKnownAnswersCorrect, RequesterStatistic.NumberPluralityAnswersCorrect, RequesterStatistic.NumberPluralityAnswersIncorrect, RequesterStatistic.NumberPluralityAnswersEvaluated, RequesterStatistic.PercentPluralityAnswersCorrect }; if (workerIds.size() == 0) { System.out.println("No workers worked on this HIT."); } else { System.out.println("Review-policy-related statistics follow."); System.out.println("These statistics are life-to-date counts for each worker who worked on this HIT."); System.out.println("These counts only include work done for the requester of the HIT."); for (String workerId : workerIds) { System.out.println("Worker " + workerId + ":"); for (RequesterStatistic statistic : statistics) { System.out.println("- " + statistic.getValue() + ": " + getStatistic(workerId, statistic)); } } } }
@SuppressWarnings("unchecked") /** * Prints the submitted results of a HIT when provided with a HIT ID. * @param hitId The HIT ID of the HIT to be retrieved. */ public void reviewAnswers(String hitId) { Assignment[] assignments = service.getAllAssignmentsForHIT(hitId); System.out.println("--[Reviewing HITs]----------"); System.out.println(" HIT Id: " + hitId); for (Assignment assignment : assignments) { //Only assignments that have been submitted will contain answer data if (assignment.getAssignmentStatus() == AssignmentStatus.Submitted) { //By default, answers are specified in XML String answerXML = assignment.getAnswer(); //Calling a convenience method that will parse the answer XML and extract out the question/answer pairs. QuestionFormAnswers qfa = RequesterService.parseAnswers(answerXML); List<QuestionFormAnswersType.AnswerType> answers = (List<QuestionFormAnswersType.AnswerType>) qfa.getAnswer(); for (QuestionFormAnswersType.AnswerType answer : answers) { String assignmentId = assignment.getAssignmentId(); String answerValue = RequesterService.getAnswerValue(assignmentId, answer); if (answerValue != null) { System.out.println("Got an answer \"" + answerValue + "\" from worker " + assignment.getWorkerId() + "."); } } //Approving the assignment. service.approveAssignment(assignment.getAssignmentId(), "Well Done!"); System.out.println("Approved."); } } System.out.println("--[End Reviewing HITs]----------"); }
@Override public void awardBonus( double amount, edu.umass.cs.surveyman.analyses.SurveyResponse sr, Survey survey) { String name = "awardBonus"; int waitTime = 1; while (true){ try { Record r = getRecord(survey); Runner.LOGGER.info("All tasks for this record:" + r.getAllTasks().length); if (r.getAllTasks().length==0){ Runner.LOGGER.info("No tasks for record " + r.rid); return; } for (ITask task : r.getAllTasks()) { Assignment[] assignments = service.getAllAssignmentsForHIT(task.getTaskId()); Runner.LOGGER.info("all assignments for this record:" + assignments.length); String assignmentId = ""; for (Assignment a : assignments) { if (a.getWorkerId().equals(sr.getSrid())) { service.grantBonus(sr.getSrid(), amount, assignmentId, "For partial work completed."); Runner.LOGGER.info(String.format("Granted worker %s bonus %f for assignment %s in survey %s" , sr.getSrid(), amount, assignmentId, survey.sourceName)); } } } } catch (InternalServiceException ise) { LOGGER.warn(format("{0} {1}", name, ise)); if (overTime(name, waitTime)){ return; } chill(waitTime); waitTime = 2 * waitTime; } catch (SurveyException | IOException e) { e.printStackTrace(); } } }
@SuppressWarnings("unchecked") /** * Prints the submitted results of a HIT when provided with a HIT ID. * @param hitId The HIT ID of the HIT to be retrieved. */ public void reviewAnswers(ReviewHIT hit) { Assignment[] assignments = service.getAllAssignmentsForHIT(hit.id); System.out.println("--[Reviewing HITs]----------"); System.out.println("author:review=" + hit.author + ":" + hit.review); System.out.println(" HIT Id: " + hit.id); for (Assignment assignment : assignments) { String path = Utils.getTranslatedFixedFilename(filenames.get(hit.author), hit.review); System.out.println(path.substring(path.lastIndexOf('/')+1)); //Only assignments that have been submitted will contain answer data if (assignment.getAssignmentStatus() == AssignmentStatus.Submitted) { //By default, answers are specified in XML String answerXML = assignment.getAnswer(); //Calling a convenience method that will parse the answer XML and extract out the question/answer pairs. QuestionFormAnswers qfa = RequesterService.parseAnswers(answerXML); List<QuestionFormAnswersType.AnswerType> answers = (List<QuestionFormAnswersType.AnswerType>) qfa.getAnswer(); for (QuestionFormAnswersType.AnswerType answer : answers) { String assignmentId = assignment.getAssignmentId(); String answerValue = RequesterService.getAnswerValue(assignmentId, answer); if (answerValue != null) { if (answerValue.equals("emptyanswer") || answerValue.equals("fair") || answerValue.equals("nice") || answerValue.equals("poor")) { // service.rejectAssignment(assignmentId, ""); System.out.println("BAD ANSWER"); } else { // service.rejectAssignment(assignmentId, ""); System.out.println("Got an answer \"" + answerValue + "\" from worker " + assignment.getWorkerId() + "."); size += 1; } } // ReviewWriter.writeTranslatedFixedReview(answerValue, hit.author, hit.review); /** Reject bad workers here. */ // if (false) { // System.out.println("BAD WORKER"); // service.rejectAssignment(assignmentId, "Do not follow the submission instructions: Submit a result(explained below) followed by the explanation of differences."); // } } // Approving the assignment. // service.approveAssignment(assignment.getAssignmentId(), "Well Done!"); // System.out.println("Approved."); // service.disposeHIT(hit.id); // service.disableHIT(hit.id); // } } System.out.println("--[End Reviewing HITs]----------"); }
private void processCrowdTask(GortEntityManager gem, EntityManager em, App app, Traversal traversal, CrowdTask crowdTask) { List<HIT> hits = crowdTask.getHits(); if (hits == null || hits.isEmpty()) { System.out.println(String.format("CrowdTask does not have any hits. (id=%d)", crowdTask.getId())); return; } Iterator<HIT> iterator = hits.iterator(); while (iterator.hasNext()) { HIT h = iterator.next(); String hitId = h.getHitId(); if (hitId == null || hitId.isEmpty()) { continue; } // check if the hit is not in flight produce the next hit // if it has not already been produced if (Boolean.FALSE.equals(h.getInFlight())) { // if another hit has already been produced or this is the final // state of crowd sourcing don't do anything if (!iterator.hasNext() && !HIT.TYPE_RESOURCE_JUSTIFICATION.equals(h.getType())) { // produce the hit for the next stage processCompletedHIT(gem, em, app, traversal, crowdTask, h); } // continue. no need to get assignments the hit is no longer in flight // so all assignments have been received continue; } System.out.println("Getting assignments for hit: " + hitId); Assignment[] assignments = Utility.getSubmittedAssignments(parent.getService(false), hitId); if (assignments == null || assignments.length <= 0) { System.out.println("Skipping hit. Hit does not have any assignments."); continue; } for (Assignment a : assignments) { String assignmentId = a.getAssignmentId(); String xmlAnswer = a.getAnswer(); if (xmlAnswer == null || xmlAnswer.isEmpty()) { continue; } org.cmuchimps.gort.modules.dataobject.Assignment gortAssignment = gem.selectAssignment(em, hitId, assignmentId); if (gortAssignment == null) { gortAssignment = Utility.createDBAssignment(gem, em, a); gortAssignment.setHit(h); gem.updateEntity(em, gortAssignment); } System.out.println(xmlAnswer); } // refresh the hit from db with potentially new assignments em.refresh(h); if (h.getAssignments().size() >= CrowdTaskMonitor.MAX_ASSIGNMENTS) { processCompletedHIT(gem, em, app, traversal, crowdTask, h); } } }
/** * Retrieves completed assignments found on the requested page for the given HIT. * * @param hitId the Id of the HIT for which completed assignments are to be returned * @param pageNum The page of results to return. Once the assignments have been filtered, * sorted, and divided into pages, the page corresponding * to pageNum is returned as the results of the operation. * @return an array of Assignments * @throws ServiceException */ public Assignment[] getAssignmentsForHIT(String hitId, int pageNum) throws ServiceException { return this.getAssignmentsForHIT(hitId, pageNum, false); }
/** * Retrieves all submitted assignments for a HIT. * * @param hitId the Id of the HIT for which the assignments are returned * @return an array of Assignments * @throws ServiceException */ public Assignment[] getAllAssignmentsForHIT(String hitId) throws ServiceException { return getAllAssignmentsForHIT(hitId, DEFAULT_ASSIGNMENT_STATUS); }
/** * Retrieves all assignments for a HIT for which * reviewable work has been submitted. * * @param hitId the Id of the HIT for which the assignments are returned * @return an array of Assignments * @throws ServiceException */ public Assignment[] getAllSubmittedAssignmentsForHIT(String hitId) throws ServiceException { return getAllAssignmentsForHIT(hitId, SUBMITTED_ASSIGNMENT_STATUS); }
