image canvas supports video frames

Author: seveibar

src/ImageCanvas/index.js

@@ -35,12 +35,16 @@ import { useRafState } from "react-use"
 import PointDistances from "../PointDistances"
 import RegionTags from "../RegionTags"
 import RegionSelectAndTransformBoxes from "../RegionSelectAndTransformBoxes"
+import VideoOrImageCanvasBackground from "../VideoOrImageCanvasBackground"
+import useEventCallback from "use-event-callback"
 
 const useStyles = makeStyles(styles)
 
 type Props = {
  regions: Array<Region>,
- imageSrc: string,
+ imageSrc?: string,
+ videoSrc?: string,
+ videoTime?: number,
  onMouseMove?: ({ x: number, y: number }) => any,
  onMouseDown?: ({ x: number, y: number }) => any,
  onMouseUp?: ({ x: number, y: number }) => any,
@@ -73,6 +77,8 @@ const getDefaultMat = () => Matrix.from(1, 0, 0, 1, -10, -10)
 export default ({
  regions,
  imageSrc,
+ videoSrc,
+ videoTime,
  realSize,
  showTags,
  onMouseMove = p => null,
@@ -138,10 +144,19 @@ export default ({
 
  const projectRegionBox = useProjectRegionBox({ layoutParams, mat })
 
- const [image, imageLoaded] = useLoadImage(imageSrc, onImageLoaded)
+ const [imageDimensions, changeImageDimensions] = useState()
+ const imageLoaded = Boolean(imageDimensions)
+ const onVideoOrImageLoaded = useEventCallback(
+ ({ naturalWidth, naturalHeight }) => {
+ changeImageDimensions({ naturalWidth, naturalHeight })
+ }
+ )
+
+ // const [image, imageLoaded] = useLoadImage(imageSrc, onImageLoaded)
  const excludePattern = useExcludePattern()
 
  useLayoutEffect(() => {
+ if (!imageDimensions) return
  const canvas = canvasEl.current
  const { clientWidth, clientHeight } = canvas
  canvas.width = clientWidth
@@ -157,13 +172,13 @@ export default ({
  )
 
  const fitScale = Math.max(
- image.naturalWidth / (clientWidth - 20),
- image.naturalHeight / (clientHeight - 20)
+ imageDimensions.naturalWidth / (clientWidth - 20),
+ imageDimensions.naturalHeight / (clientHeight - 20)
  )
 
  const [iw, ih] = [
- image.naturalWidth / fitScale,
- image.naturalHeight / fitScale
+ imageDimensions.naturalWidth / fitScale,
+ imageDimensions.naturalHeight / fitScale
  ]
 
  layoutParams.current = {
@@ -378,22 +393,24 @@ export default ({
  }}
  >
  {showCrosshairs && <Crosshairs mousePosition={mousePosition} />}
- <RegionSelectAndTransformBoxes
- regions={regions}
- mouseEvents={mouseEvents}
- projectRegionBox={projectRegionBox}
- dragWithPrimary={dragWithPrimary}
- createWithPrimary={createWithPrimary}
- zoomWithPrimary={zoomWithPrimary}
- onBeginMovePoint={onBeginMovePoint}
- onSelectRegion={onSelectRegion}
- layoutParams={layoutParams}
- mat={mat}
- onBeginBoxTransform={onBeginBoxTransform}
- onBeginMovePolygonPoint={onBeginMovePolygonPoint}
- onAddPolygonPoint={onAddPolygonPoint}
- />
- {showTags && (
+ {imageLoaded && (
+ <RegionSelectAndTransformBoxes
+ regions={regions}
+ mouseEvents={mouseEvents}
+ projectRegionBox={projectRegionBox}
+ dragWithPrimary={dragWithPrimary}
+ createWithPrimary={createWithPrimary}
+ zoomWithPrimary={zoomWithPrimary}
+ onBeginMovePoint={onBeginMovePoint}
+ onSelectRegion={onSelectRegion}
+ layoutParams={layoutParams}
+ mat={mat}
+ onBeginBoxTransform={onBeginBoxTransform}
+ onBeginMovePolygonPoint={onBeginMovePolygonPoint}
+ onAddPolygonPoint={onAddPolygonPoint}
+ />
+ )}
+ {imageLoaded && showTags && (
  <PreventScrollToParents>
  <RegionTags
  regions={regions}
@@ -437,7 +454,15 @@ export default ({
  >
  <>
  <canvas className={classes.canvas} ref={canvasEl} />
- <img
+ <VideoOrImageCanvasBackground
+ imagePosition={imagePosition}
+ mouseEvents={mouseEvents}
+ onLoad={onVideoOrImageLoaded}
+ videoTime={videoTime}
+ videoSrc={videoSrc}
+ imageSrc={imageSrc}
+ />
+ {/* <img
  src={imageSrc}
  className={classes.image}
  {...mouseEvents}
@@ -447,7 +472,7 @@ export default ({
  width: imagePosition.bottomRight.x - imagePosition.topLeft.x,
  height: imagePosition.bottomRight.y - imagePosition.topLeft.y
  }}
- />
+ /> */}
  </>
  </PreventScrollToParents>
  <div className={classes.zoomIndicator}>

src/ImageCanvas/styles.js

@@ -1,7 +1,6 @@
 import { grey } from "@material-ui/core/colors"
 
 export default {
- image: { zIndex: 0, position: "absolute" },
  canvas: { width: "100%", height: "100%", position: "relative", zIndex: 1 },
  zoomIndicator: {
  position: "absolute",

src/RegionTags/index.js

@@ -45,6 +45,7 @@ export const RegionTags = ({
  if (region.locked) {
  return (
  <div
+ key={region.id}
  style={{
  position: "absolute",
  ...coords,
@@ -72,6 +73,7 @@ export const RegionTags = ({
  }
  return (
  <div
+ key={region.id}
  style={{
  position: "absolute",
  ...coords,

src/SettingsDialog/index.js

@@ -12,7 +12,7 @@ import { useSettings } from "../SettingsProvider"
 export const SettingsDialog = ({ open, onClose }) => {
  const settings = useSettings()
  return (
- <Dialog open={open} onClose={onClose}>
+ <Dialog open={open || false} onClose={onClose}>
  <DialogTitle>Settings</DialogTitle>
  <DialogContent style={{ minWidth: 400 }}>
  <Survey

src/VideoOrImageCanvasBackground/index.js

@@ -0,0 +1,88 @@
+// @flow weak
+
+import React, { useRef, useEffect, useMemo } from "react"
+import { styled } from "@material-ui/core/styles"
+import useEventCallback from "use-event-callback"
+
+const Video = styled("video")({
+ zIndex: 0,
+ position: "absolute"
+})
+
+const StyledImage = styled("img")({
+ zIndex: 0,
+ position: "absolute"
+})
+
+export default ({
+ imagePosition,
+ mouseEvents,
+ videoTime = 0,
+ videoSrc,
+ imageSrc,
+ onLoad
+}) => {
+ const videoRef = useRef()
+ const imageRef = useRef()
+
+ useEffect(() => {
+ if (videoRef.current) {
+ videoRef.current.currentTime = videoTime / 1000
+ }
+ }, [videoTime])
+
+ const onLoadedVideoMetadata = useEventCallback(event => {
+ const videoElm = event.currentTarget
+ videoElm.currentTime = videoTime / 1000
+ if (onLoad)
+ onLoad({
+ naturalWidth: videoElm.videoWidth,
+ naturalHeight: videoElm.videoHeight,
+ videoElm: videoElm
+ })
+ })
+ const onImageLoaded = useEventCallback(event => {
+ const imageElm = event.currentTarget
+ if (onLoad)
+ onLoad({
+ naturalWidth: imageElm.naturalWidth,
+ naturalHeight: imageElm.naturalHeight,
+ imageElm
+ })
+ })
+
+ const stylePosition = useMemo(
+ () => ({
+ left: imagePosition.topLeft.x,
+ top: imagePosition.topLeft.y,
+ width: imagePosition.bottomRight.x - imagePosition.topLeft.x,
+ height: imagePosition.bottomRight.y - imagePosition.topLeft.y
+ }),
+ [
+ imagePosition.topLeft.x,
+ imagePosition.topLeft.y,
+ imagePosition.bottomRight.x,
+ imagePosition.bottomRight.y
+ ]
+ )
+
+ if (!videoSrc && !imageSrc) return "No imageSrc or videoSrc provided"
+
+ return imageSrc ? (
+ <StyledImage
+ {...mouseEvents}
+ src={imageSrc}
+ ref={imageRef}
+ style={stylePosition}
+ onLoad={onImageLoaded}
+ />
+ ) : (
+ <Video
+ {...mouseEvents}
+ ref={videoRef}
+ style={stylePosition}
+ onLoadedMetadata={onLoadedVideoMetadata}
+ src={videoSrc}
+ />
+ )
+}