{"type":"doc","content":[{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{"style":{"value":"none"}},"macroMetadata":{"macroId":{"value":"f8415bba-c19b-4b8d-98e4-809815da4c21"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}},"localId":"da24fe8d-7187-48a2-9d83-ad49a9077820"}},{"type":"heading","attrs":{"level":1},"content":[{"text":"TODOs (updated 12/10/2024)","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Flow chart of data transfer and processing for both Field and SemiF (","type":"text"},{"type":"mention","attrs":{"id":"5f08cfa2e5efc1001b836c6d","localId":"2fe85889-2534-46d6-aee2-d7be7c3878c7","text":"@Matthew Kutugata"}},{"text":") could be a general flowchart placed in the main preprocessing page. ","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"type":"mention","attrs":{"id":"712020:a281eea0-1070-45ab-ab72-22d8bd094e3a","localId":"581f16bd-4fcb-446f-9940-3772f01dabca","text":"@Courtney Belcher"}},{"text":" ","type":"text"},{"type":"mention","attrs":{"id":"62aaaa8df8bd790069d914ec","localId":"227c67ee-ab4e-49d7-bec3-b65005f26508","text":"@Courtney Belcher (Unlicensed)"}},{"text":" could give it a shot","type":"text"}]}]}]}]}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"Backlog Spreadsheet:","type":"text","marks":[{"type":"strong"}]},{"type":"inlineExtension","attrs":{"extensionType":"com.atlassian.confluence.macro.core","extensionKey":"lref-gdrive-file","parameters":{"macroParams":{"url":{"value":"https://docs.google.com/spreadsheets/d/18fwdSJxMy1WjCBCSCTb3uERj5D32H5jmBEh5dVpvrog/edit?gid=1066600493#gid=1066600493"}},"macroMetadata":{"macroId":{"value":"d3621695-7ca2-41b5-be7f-161aa3875884"},"schemaVersion":{"value":"1"},"placeholder":[{"type":"icon","data":{"url":"https://lref.bilith.com/static/img/gdrive/macro_ref_link.png"}}],"title":"Google Drive Live Link"}},"localId":"827daa97-667a-46e4-aa5c-f8d83fe2e8e3"}}]},{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"** THIS PAGE IS BASED ON BBV3.0 - SONY CAMERA **","type":"text","marks":[{"type":"strong"}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"1. Downloading a batch","type":"text","marks":[{"type":"annotation","attrs":{"annotationType":"inlineComment","id":"ce808671-17ab-4bfa-bc19-c19d6b8a0e45"}}]}]},{"type":"paragraph","content":[{"text":"When downloading a Semifield batch, It copies over to the local user from the NFS Lockers.","type":"text","marks":[{"type":"annotation","attrs":{"annotationType":"inlineComment","id":"ce808671-17ab-4bfa-bc19-c19d6b8a0e45"}}]}]},{"type":"codeBlock","content":[{"text":"cd ..\ncd psa_images/semifield_tools/\nscreen -R \npython3 copy_from_lockers_test.py ","type":"text"}]},{"type":"paragraph","content":[{"text":"Semifield batches can take anywhere from a few minutes to half an hour to copy over, depending on the batch size. It is important to open a new screen for copying. Only to copy one or two batches at a time within their respective screens to avoid disk space problems. ","type":"text"}]},{"type":"paragraph","content":[{"text":"To exit the screen, use CTRL+A D","type":"text"}]},{"type":"paragraph","content":[{"text":"Checking in to see if the copying of a batch has been completed by using this string of commands:","type":"text"}]},{"type":"codeBlock","content":[{"text":"cd psa_images/semifield_tools/\nscreen -r ","type":"text"}]},{"type":"paragraph","content":[{"text":"If the batch has either been completed copying or there has been an error copying, there will be an input line. If no input line, the process has not completed.","type":"text"}]},{"type":"paragraph","content":[{"text":"To exit the screen, use CTRL+A D","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"wide","width":764,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":2048,"alt":"Screen Recording 2025-02-03 112423.mp4","id":"7590a984-2537-41de-a7e9-9dbb7596092c","collection":"contentId-743669767","type":"file","height":1152}},{"type":"caption","content":[{"text":"Example of Semifield batch copying","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"1.1 Checking if the batch was completely copied","type":"text"}]},{"type":"paragraph","content":[{"text":"Check if the copied batch was brought to the local user entirely. We must know the","type":"text"},{"text":" # of images in the original batch","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"codeBlock","content":[{"text":"ls /mnt/research-projects/s/screberg/longterm_images2/semifield-upload//*.ARW | wc -l","type":"text"}]},{"type":"paragraph","content":[{"text":"Sometimes this command may need to be run two or three times before it output the correct value.","type":"text"}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"Check the ","type":"text"},{"text":"# of images in the local directory ","type":"text","marks":[{"type":"strong"}]},{"text":"after download has completed:","type":"text"}]},{"type":"codeBlock","content":[{"text":"ls /home/psa_images/temp_data/semifield-upload//*.ARW | wc -l","type":"text"}]},{"type":"paragraph","content":[{"text":"(Moving files from longterm storage to the local working directory, before you run the the pipeline). ","type":"text"}]},{"type":"paragraph","content":[{"text":"*The output should be equal to the above command; should be 0 after running the pipeline","type":"text"}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"After running the pipeline, check if all the ","type":"text"},{"text":"ARW files have converted to JPGs ","type":"text","marks":[{"type":"strong"}]},{"text":"and have moved to the longterm “developed” storage:","type":"text"}]},{"type":"codeBlock","content":[{"text":"ls /mnt/research-projects/s/screberg/longterm_images2/semifield-developed-images//images/*.jpg | wc -l","type":"text"}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"Here is an example of checking if batches were copied","type":"text","marks":[{"type":"strong"},{"type":"link","attrs":{"href":"https://precision-sustainable-ag.atlassian.net/wiki/spaces/IR2/pages/743669767/SemiField+Preprocessing#Example%3A-MD_2024-07-02"}}]}]},{"type":"paragraph"},{"type":"heading","attrs":{"level":2},"content":[{"text":"2. Color Profiling in RawTherapee","type":"text"}]},{"type":"paragraph","content":[{"text":"Launch RawTherapee:","type":"text"}]},{"type":"codeBlock","content":[{"text":"./RawTherapee_5.8.AppImage","type":"text"}]},{"type":"paragraph","content":[{"text":"RawTherapee will launch in a separate window and opens to the most recently worked on profile. Select the appropriate batch and allow it to load. Scroll through and note if some images are out of the ordinary (i.e. too bright, too dark, out of focus, Amiga in the frame, etc.). When creating a color profile, the goal is to have every image look similarly to the rest of the batch.","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center","width":760,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":1532,"alt":"Step1.mp4","id":"dce70e32-5b14-48b9-b40a-5fe0bab84423","collection":"contentId-743669767","type":"file","height":1004}},{"type":"caption","content":[{"text":"Scan, Find Color Checker, Choose Image","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"2.1 Look for a Color Checker","type":"text"}]},{"type":"paragraph","content":[{"text":"Using an image with a color checker, if one is available, choose the most average looking picture (lighting and color-wise). If there is no biomass present, choose an average image with a color checker.","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center","width":248,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":304,"alt":"Screenshot 2024-12-06 144011.png","id":"79f05ac9-70ad-4320-a4cb-73f9b22f4660","collection":"contentId-743669767","type":"file","height":453}},{"type":"caption","content":[{"text":"Semifield Color Checker","type":"text"}]}]},{"type":"paragraph","content":[{"text":"*In Semifield applications, a smaller Color Checker lanyard is typically used. The Field Color Checker might be used instead, but the smaller one is preferred for this application. Either is acceptable.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"2.2 Bring in a Recent profile & Color Picker","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center","width":757,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":1536,"alt":"Step2_1 - Trim.mp4","id":"56b2aa69-277e-41aa-abb8-25b6225f9808","collection":"contentId-743669767","type":"file","height":998}},{"type":"caption","content":[{"text":"Bring in Recent Profile and Color Picker","type":"text"}]}]},{"type":"paragraph","content":[{"text":"To bring in a recent profile for the location being worked on. At the top right of the scree, locate the folder icon. This will open a window containing a list of saved profiles within the local user. Choose the most recent date in relation to the image being worked on.","type":"text"}]},{"type":"paragraph","content":[{"text":"Once the previous profile is loaded in and applied, use the color picker tool. The Color Picker tool is found on the tool bar above the working image or use the White Balance tab on the panel to the right. Choose “Pick”.","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center","width":380,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":176,"alt":"d5c9b176-2e22-4aaa-8702-fdc6675e59e7.png","id":"3e97c2dc-a5e7-4201-92fc-06d262de1696","collection":"contentId-743669767","type":"file","height":152}},{"type":"caption","content":[{"text":"Semifield Color Checker","type":"text"}]}]},{"type":"paragraph","content":[{"text":"Only one square on the Color Checker is used for in this application. The gray square residing two squares above the black is the color to use. Use this gray for every profile. Choose the hand icon to return to normal cursor mode.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"2.3 Adjust exposure & saturation","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center","width":757,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":1536,"alt":"Step2_2 - Trim.mp4","id":"ec67cea7-3cfb-4321-a169-c0e6abd42159","collection":"contentId-743669767","type":"file","height":998}},{"type":"caption","content":[{"text":"Adjust Exposure & Saturation","type":"text"}]}]},{"type":"paragraph","content":[{"text":"Now, navigate into the Exposure tab. This is the only tab used in preprocessing, aside from the tab to access the Color Picker. There are only a few helpful settings to toggle here.","type":"text"}]},{"type":"paragraph","content":[{"text":"The goal is to create an image that looks most closely like a real-life view of these plants. The human eye is the most accurate check, but it can be helpful to reference the color graph located on the top left of the scree. It shifts when settings are changed. A good rule of thumb is for the peak of the three curves to land near this region of the graph.","type":"text"}]},{"type":"mediaSingle","attrs":{"layout":"center","width":422,"widthType":"pixel"},"content":[{"type":"media","attrs":{"width":390,"alt":"graph.mp4","id":"04ec898d-582e-42ab-a4e8-335579fdcd74","collection":"contentId-743669767","type":"file","height":282}},{"type":"caption","content":[{"text":"Color Graph - RawTherapee","type":"text"}]}]},{"type":"paragraph","content":[{"text":"*Laptop screens and Monitor settings will impact perception of images. Try to use the same screen when creating color profiles.","type":"text","marks":[{"type":"em"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Exposure Compensation & Highlight Compression","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"These two go hand-in-hand. Exposure compensation should always be set to slightly more than Highlight Compression","type":"text"}]}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Lightness","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Has a similar effect to Exposure Compensation","type":"text"}]}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Saturation","type":"text","marks":[{"type":"strong"}]}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Toggle keeping in mind the intensity of colors on the color checker.","type":"text"}]}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"3. Running the pipeline ","type":"text"}]},{"type":"paragraph","marks":[{"type":"alignment","attrs":{"align":"center"}}],"content":[{"text":"** Do not run >3 batches in the pipeline at once. **","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Profiling can be done and batches can be prepared for the pipeline, but wait to run them until another batch finished.","type":"text"}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"After the color profile has been saved, send it through the pipeline. Be sure to be navigated to the Semifield folder and make a new screen, use uppercase r (-R). If returning to an already existing screen, use lowercase r (-r) with the appropriate screen name. ","type":"text"}]},{"type":"codeBlock","content":[{"text":"cd ..\ncd psa_images/semifield_tools/\nscreen -R \npython3 semifield_CLOUDY_pipeline.py ","type":"text"}]},{"type":"paragraph","content":[{"text":"To exit the screen, use CTRL+A D","type":"text"}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"Check on a batch sent into the pipeline by using this string of commands:","type":"text"}]},{"type":"codeBlock","content":[{"text":"cd psa_images/semifield_tools/\nscreen -r ","type":"text"}]},{"type":"paragraph","content":[{"text":"If the pipeline has either been completed running or there has been an error running it, there will be an input line. If there is no input line, the process has not completed.","type":"text"}]},{"type":"paragraph","content":[{"text":"To exit the screen, use CTRL+A D","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"3.1 Checking the success of the pipeline","type":"text"}]},{"type":"paragraph","content":[{"text":"If there is an indication the pipeline is finished running, crosscheck the # of images in the original batch with the ","type":"text"},{"text":"# of JPGs and PP3s created.","type":"text","marks":[{"type":"strong"}]},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"We can use both of these lines respectively:","type":"text"}]},{"type":"codeBlock","content":[{"text":"ls /mnt/research-projects/s/screberg/longterm_images2/semifield-developed-images//images/*.jpg | wc -l","type":"text"}]},{"type":"codeBlock","content":[{"text":"ls /mnt/research-projects/s/screberg/longterm_images2/semifield-developed-images//images/*.pp3 | wc -l","type":"text"}]},{"type":"paragraph","content":[{"text":"And compare to the ","type":"text"},{"text":"# of ARWs ","type":"text","marks":[{"type":"strong"}]},{"text":"using this:","type":"text"}]},{"type":"codeBlock","content":[{"text":"ls /mnt/research-projects/s/screberg/longterm_images2/semifield-upload//*.ARW | wc -l","type":"text"}]},{"type":"paragraph","content":[{"text":"As long as the output # are equivalent, the color profile has successfully been applied to every image in the batch.","type":"text"}]},{"type":"paragraph","content":[{"text":"If the # of JPGs and PP3s are not equivalent to the original # of images, try running the pipeline again within the same screen. Double check the name of the batch that was typed in, this is where errors are likely to happen. Also, be sure to be navigated to the Semifield folder. If navigated to the Field folder, the pipeline will not be successful. ","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"4. Update Backlog Spreadsheet","type":"text"}]},{"type":"paragraph","content":[{"text":"Change the progress column to the appropriate term to indicate where that batch is in the process. Make any notes/check box when done, if applicable. ","type":"text"}]},{"type":"paragraph","content":[{"text":"When entire batch is fully completed, the row can be hidden by right clicking on the row #, then click hide. ","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"5. Troubleshooting","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"If developed jpgs are not in the longterm storage, check to see if they’re still in the local temp_data outputs folder:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Check for jpgs:","type":"text"}]}]}]},{"type":"codeBlock","content":[{"text":"ls /home/psa_images/temp_data/semifield-outputs/MD_2024-07-26/images/*.jpg | wc -l","type":"text"}]},{"type":"paragraph","content":[{"text":"check for same number of .pp3 files (how you know they’ve been developed):","type":"text"}]},{"type":"codeBlock","content":[{"text":"ls /home/psa_images/temp_data/semifield-outputs/MD_2024-07-26/images/*.pp3 | wc -l","type":"text"}]},{"type":"paragraph","content":[{"text":"If they’re both present and have the same number, we know they’ve been successfully preprpocessed, but may not have been successfully move to the longterm locker storage location. ","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"If there is an “Assertion Error”, try going into RawTherapee, and create & save the profile again. ","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"6. Common & Useful Commands","type":"text"}]},{"type":"paragraph","content":[{"text":"Available batches:","type":"text","marks":[{"type":"strong"}]}]},{"type":"codeBlock","content":[{"text":"ls /mnt/research-projects/s/screberg/longterm_images2/semifield-upload/ -lh","type":"text"}]},{"type":"paragraph","content":[{"text":"# of images in batch:","type":"text","marks":[{"type":"strong"}]}]},{"type":"codeBlock","content":[{"text":"ls /mnt/research-projects/s/screberg/longterm_images2/semifield-upload//*.ARW | wc -l","type":"text"}]},{"type":"paragraph","content":[{"text":"# of JPGs processed:","type":"text","marks":[{"type":"strong"}]}]},{"type":"codeBlock","content":[{"text":"ls /mnt/research-projects/s/screberg/longterm_images2/semifield-developed-images//images/*.jpg | wc -l","type":"text"}]},{"type":"paragraph","content":[{"text":"# of PP3 files created:","type":"text","marks":[{"type":"strong"}]}]},{"type":"codeBlock","content":[{"text":"ls /mnt/research-projects/s/screberg/longterm_images2/semifield-developed-images//images/*.pp3 | wc -l","type":"text"}]},{"type":"paragraph"},{"type":"panel","attrs":{"panelType":"note"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Example: ","type":"text","marks":[{"type":"strong"}]},{"text":"MD_2024-07-02","type":"text"}]},{"type":"paragraph","content":[{"text":"Step 1: number of raw images in storage locker: ","type":"text"}]},{"type":"codeBlock","content":[{"text":" ls /mnt/research-projects/s/screberg/longterm_images2/semifield-upload/MD_2024-07-02/*.ARW | wc -l","type":"text"}]},{"type":"paragraph","content":[{"text":"Output: 560","type":"text"}]}]},{"type":"panel","attrs":{"panelType":"note"},"content":[{"type":"paragraph","content":[{"text":"Step 2: number of raw images brought to the local user from the storage locker:","type":"text"}]},{"type":"codeBlock","content":[{"text":"ls /home/psa_images/temp_data/semifield-upload/MD_2024-07-02/*.ARW | wc -l","type":"text"}]},{"type":"paragraph","content":[{"text":"Output (before running the pipeline): 560","type":"text"}]},{"type":"paragraph","content":[{"text":"Output (after running the pipeline): 0","type":"text"}]},{"type":"paragraph","content":[{"text":"The output after running the pipeline is 0 because images are cleared from the local user after being pulled into the pipeline. The batch name will remain in the local user but should not have any images left within it.","type":"text"}]}]},{"type":"panel","attrs":{"panelType":"note"},"content":[{"type":"paragraph","content":[{"text":"Step 3: number of new jpgs developed (made from raws):","type":"text"}]},{"type":"codeBlock","content":[{"text":"ls /mnt/research-projects/s/screberg/longterm_images2/semifield-developed-images/MD_2024-07-02/images/*.jpg | wc -l","type":"text"}]},{"type":"paragraph","content":[{"text":"output (after running the pipeline): 560","type":"text"}]}]}],"version":1}

Browser not supported