lukas/shenzhen-solitaire
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made image detection work

Browse Source
This commit is contained in:
Lukas Wölfer
2020-02-05 01:51:30 +01:00
parent 7e18f1db1c
commit c1026c0640
7 changed files with 208 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
shenzhen_solitaire/card_detection/board_parser.py
View File

@@ -4,34 +4,49 @@ import numpy as np
from .configuration import Configuration from .configuration import Configuration
from ..board import Board from ..board import Board
from . import card_finder from . import card_finder
import cv2
from typing import Iterable, Any, List
import itertools
def parse_board(image: np.ndarray, conf: Configuration) -> Board: def parse_board(image: np.ndarray, conf: Configuration) -> Board:
"""Parse a screenshot of the game, using a given configuration""" """Parse a screenshot of the game, using a given configuration"""
fake_adjustments = conf.field_adjustment
fake_adjustments.x -= 5
fake_adjustments.y -= 5
fake_adjustments.h += 10
fake_adjustments.w += 10
row_count = 13
column_count = 8
def grouper(iterable: Iterable[Any], groupsize: int, fillvalue: Any = None) -> Iterable[Any]:
"Collect data into fixed-length chunks or blocks"
args = [iter(iterable)] * groupsize
return itertools.zip_longest(*args, fillvalue=fillvalue)
squares = card_finder.get_field_squares( squares = card_finder.get_field_squares(
image, conf.field_adjustment, count_x=13, count_y=8) image, conf.field_adjustment, count_x=row_count, count_y=column_count
squares = [card_finder.simplify(square)[0] for square in squares] )
square_rows = [squares[13 * i:13 * (i + 1)] for i in range(8)] grouped_squares = grouper(squares, row_count)
empty_square = np.full( result = Board()
shape=(conf.field_adjustment.w, for group_index, square_group in enumerate(grouped_squares):
conf.field_adjustment.h), group_field = []
fill_value=card_finder.GREYSCALE_COLOR[card_finder.Cardcolor.Background], for index, square in enumerate(square_group):
dtype=np.uint8) best_val = None
assert empty_square.shape == squares[0].shape best_name = None
result: Board = Board() for template, name in conf.catalogue:
for row_id, square_row in enumerate(square_rows): res = cv2.matchTemplate(square, template, cv2.TM_CCOEFF_NORMED)
for square in square_row: min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
fitting_square, _ = card_finder.find_square( if best_val is None or max_val > best_val:
square, [empty_square] + [x[0] for x in conf.catalogue]) best_val = max_val
if np.array_equal(fitting_square, empty_square): best_name = name
print("empty") assert best_name is not None
break group_field.append(best_name)
for cat_square, cardtype in conf.catalogue:
if np.array_equal(fitting_square, cat_square): # print(f"\t{best_val}: {best_name}")
print(cardtype) # cv2.imshow("Catalogue", cv2.resize(square, (500, 500)))
result.field[row_id].append(cardtype) # cv2.waitKey()
break
else: result.field[group_index] = group_field
print("did not find image")
return result return result

112
shenzhen_solitaire/card_detection/card_finder.py
View File

@@ -3,25 +3,23 @@
from typing import List, Tuple, Optional, Dict from typing import List, Tuple, Optional, Dict
import enum import enum
import itertools import itertools
import numpy as np # type: ignore import numpy as np
import cv2 # type: ignore import cv2
from .adjustment import Adjustment, get_square from .adjustment import Adjustment, get_square
from ..board import Card, NumberCard, SpecialCard from ..board import Card, NumberCard, SpecialCard
def _extract_squares(image: np.ndarray, def _extract_squares(
squares: List[Tuple[int, image: np.ndarray, squares: List[Tuple[int, int, int, int]]
int, ) -> List[np.ndarray]:
int, return [
int]]) -> List[np.ndarray]: image[square[1] : square[3], square[0] : square[2]].copy() for square in squares
return [image[square[1]:square[3], square[0]:square[2]].copy() ]
for square in squares]
def get_field_squares(image: np.ndarray, def get_field_squares(
adjustment: Adjustment, image: np.ndarray, adjustment: Adjustment, count_x: int, count_y: int
count_x: int, ) -> List[np.ndarray]:
count_y: int) -> List[np.ndarray]:
"""Return all squares in the field, according to the adjustment""" """Return all squares in the field, according to the adjustment"""
squares = [] squares = []
for index_x, index_y in itertools.product(range(count_y), range(count_x)): for index_x, index_y in itertools.product(range(count_y), range(count_x)):
@@ -31,64 +29,43 @@ def get_field_squares(image: np.ndarray,
class Cardcolor(enum.Enum): class Cardcolor(enum.Enum):
"""Relevant colors for different types of cards""" """Relevant colors for different types of cards"""
Bai = (65, 65, 65) Bai = (65, 65, 65)
Black = (0, 0, 0) Black = (0, 0, 0)
Red = (22, 48, 178) Red = (22, 48, 178)
Green = (76, 111, 19) Green = (76, 111, 19)
Background = (178, 194, 193) Background = (178, 194, 193)
def _find_single_square(
GREYSCALE_COLOR = { search_square: np.ndarray, template_square: np.ndarray
Cardcolor.Bai: 50, ) -> Tuple[int, Tuple[int, int]]:
Cardcolor.Black: 100,
Cardcolor.Red: 150,
Cardcolor.Green: 200,
Cardcolor.Background: 250}
def simplify(image: np.ndarray) -> Tuple[np.ndarray, Dict[Cardcolor, int]]:
"""Reduce given image to the colors in Cardcolor"""
result_image: np.ndarray = np.zeros(
(image.shape[0], image.shape[1]), np.uint8)
result_dict: Dict[Cardcolor, int] = {c: 0 for c in Cardcolor}
for pixel_x, pixel_y in itertools.product(
range(result_image.shape[0]),
range(result_image.shape[1])):
pixel = image[pixel_x, pixel_y]
best_color: Optional[Tuple[Cardcolor, int]] = None
for color in Cardcolor:
mse = sum((x - y) ** 2 for x, y in zip(color.value, pixel))
if not best_color or best_color[1] > mse: #pylint: disable=E1136
best_color = (color, mse)
assert best_color
result_image[pixel_x, pixel_y] = GREYSCALE_COLOR[best_color[0]]
result_dict[best_color[0]] += 1
return (result_image, result_dict)
def _find_single_square(search_square: np.ndarray,
template_square: np.ndarray) -> Tuple[int, Tuple[int, int]]:
assert search_square.shape[0] >= template_square.shape[0] assert search_square.shape[0] >= template_square.shape[0]
assert search_square.shape[1] >= template_square.shape[1] assert search_square.shape[1] >= template_square.shape[1]
best_result: Optional[Tuple[int, Tuple[int, int]]] = None best_result: Optional[Tuple[int, Tuple[int, int]]] = None
for margin_x, margin_y in itertools.product( for margin_x, margin_y in itertools.product(
range(search_square.shape[0], template_square.shape[0] - 1, -1), range(search_square.shape[0], template_square.shape[0] - 1, -1),
range(search_square.shape[1], template_square.shape[1] - 1, -1)): range(search_square.shape[1], template_square.shape[1] - 1, -1),
search_region = search_square[margin_x - ):
template_square.shape[0]:margin_x, margin_y - search_region = search_square[
template_square.shape[1]:margin_y] margin_x - template_square.shape[0] : margin_x,
margin_y - template_square.shape[1] : margin_y,
]
count = cv2.countNonZero(search_region - template_square) count = cv2.countNonZero(search_region - template_square)
if not best_result or count < best_result[0]: #pylint: disable=E1136 if not best_result or count < best_result[0]: # pylint: disable=E1136
best_result = ( best_result = (
count, count,
(margin_x - template_square.shape[0], (
margin_y - template_square.shape[1])) margin_x - template_square.shape[0],
margin_y - template_square.shape[1],
),
)
assert best_result assert best_result
return best_result return best_result
def find_square(search_square: np.ndarray, def find_square(
squares: List[np.ndarray]) -> Tuple[np.ndarray, int]: search_square: np.ndarray, squares: List[np.ndarray]
) -> Tuple[np.ndarray, int]:
"""Compare all squares in squares with search_square, return best matching one. """Compare all squares in squares with search_square, return best matching one.
Requires all squares to be simplified.""" Requires all squares to be simplified."""
best_set = False best_set = False
@@ -104,24 +81,24 @@ def find_square(search_square: np.ndarray,
return (best_square, best_count) return (best_square, best_count)
def catalogue_cards(squares: List[np.ndarray] def catalogue_cards(squares: List[np.ndarray]) -> List[Tuple[np.ndarray, Card]]:
) -> List[Tuple[np.ndarray, Card]]:
"""Run manual cataloging for given squares""" """Run manual cataloging for given squares"""
cv2.namedWindow("Catalogue", cv2.WINDOW_NORMAL) cv2.namedWindow("Catalogue", cv2.WINDOW_NORMAL)
cv2.waitKey(1) cv2.waitKey(1)
result: List[Tuple[np.ndarray, Card]] = [] result: List[Tuple[np.ndarray, Card]] = []
print( print("Card ID is [B]ai, [Z]hong, [F]a, [H]ua, [R]ed, [G]reen, [B]lack")
"Card ID is [B]ai, [Z]hong, [F]a, [H]ua, [R]ed, [G]reen, [B]lack")
print("Numbercard e.g. R3") print("Numbercard e.g. R3")
special_card_map = { special_card_map = {
'b': SpecialCard.Bai, "b": SpecialCard.Bai,
'z': SpecialCard.Zhong, "z": SpecialCard.Zhong,
'f': SpecialCard.Fa, "f": SpecialCard.Fa,
'h': SpecialCard.Hua} "h": SpecialCard.Hua,
}
suit_map = { suit_map = {
'r': NumberCard.Suit.Red, "r": NumberCard.Suit.Red,
'g': NumberCard.Suit.Green, "g": NumberCard.Suit.Green,
'b': NumberCard.Suit.Black} "b": NumberCard.Suit.Black,
}
for square in squares: for square in squares:
while True: while True:
cv2.imshow("Catalogue", cv2.resize(square, (500, 500))) cv2.imshow("Catalogue", cv2.resize(square, (500, 500)))
@@ -137,10 +114,11 @@ def catalogue_cards(squares: List[np.ndarray]
continue continue
if not card_id[1].isdigit(): if not card_id[1].isdigit():
continue continue
if card_id[1] == '0': if card_id[1] == "0":
continue continue
card_type = NumberCard(number=int( card_type = NumberCard(
card_id[1]), suit=suit_map[card_id[0]]) number=int(card_id[1]), suit=suit_map[card_id[0]]
)
else: else:
continue continue
assert card_type is not None assert card_type is not None

87
shenzhen_solitaire/card_detection/configuration.py
View File

@@ -4,6 +4,8 @@ import json
from typing import List, Tuple, Dict from typing import List, Tuple, Dict
import io import io
import dataclasses import dataclasses
import tempfile
import cv2
import numpy as np import numpy as np
from . import adjustment from . import adjustment
@@ -13,15 +15,16 @@ from .. import board
class Configuration: class Configuration:
"""Configuration for solitaire cv""" """Configuration for solitaire cv"""
ADJUSTMENT_FILE_NAME = 'adjustment.json'
TEMPLATES_DIRECTORY = 'templates'
def __init__(self, ADJUSTMENT_FILE_NAME = "adjustment.json"
TEMPLATES_DIRECTORY = "templates"
def __init__(
self,
adj: adjustment.Adjustment, adj: adjustment.Adjustment,
catalogue: List[Tuple[np.ndarray, catalogue: List[Tuple[np.ndarray, board.Card]],
board.Card]], meta: Dict[str, str],
meta: Dict[str, ) -> None:
str]) -> None:
self.field_adjustment = adj self.field_adjustment = adj
self.catalogue = catalogue self.catalogue = catalogue
self.meta = meta self.meta = meta
@@ -29,67 +32,73 @@ class Configuration:
def save(self, filename: str) -> None: def save(self, filename: str) -> None:
"""Save configuration to zip archive""" """Save configuration to zip archive"""
zip_stream = io.BytesIO() zip_stream = io.BytesIO()
with zipfile.ZipFile(zip_stream, "w") as zip_file: with zipfile.ZipFile(zip_stream, "w") as zip_file:
zip_file.writestr( zip_file.writestr(
self.ADJUSTMENT_FILE_NAME, json.dumps( self.ADJUSTMENT_FILE_NAME,
dataclasses.asdict( json.dumps(dataclasses.asdict(self.field_adjustment)),
self.field_adjustment))) )
counter = 0 counter = 0
extension = ".png"
for square, card in self.catalogue: for square, card in self.catalogue:
counter += 1 counter += 1
file_stream = io.BytesIO() fd, myfile = tempfile.mkstemp()
np.save( cv2.imwrite(myfile + extension, square)
file_stream,
card_finder.simplify(square)[0],
allow_pickle=False)
file_name = "" file_name = ""
if isinstance(card, board.SpecialCard): if isinstance(card, board.SpecialCard):
file_name = f's{card.value}-{card.name}-{counter}.npy' file_name = f"s{card.value}-{card.name}-{counter}{extension}"
elif isinstance(card, board.NumberCard): elif isinstance(card, board.NumberCard):
file_name = f'n{card.suit.value}{card.number}'\ file_name = (
f'-{card.suit.name}-{counter}.npy' f"n{card.suit.value}{card.number}"
f"-{card.suit.name}-{counter}{extension}"
)
else: else:
raise AssertionError() raise AssertionError()
zip_file.writestr( zip_file.write(myfile + extension, arcname=f"{self.TEMPLATES_DIRECTORY}/{file_name}")
self.TEMPLATES_DIRECTORY + f"/{file_name}",
file_stream.getvalue())
with open(filename, 'wb') as zip_archive: with open(filename, "wb") as zip_archive:
zip_archive.write(zip_stream.getvalue()) zip_archive.write(zip_stream.getvalue())
@staticmethod @staticmethod
def load(filename: str) -> 'Configuration': def load(filename: str) -> "Configuration":
"""Load configuration from zip archive""" """Load configuration from zip archive"""
def _parse_file_name(card_filename: str) -> board.Card: def _parse_file_name(card_filename: str) -> board.Card:
assert card_filename.startswith( assert card_filename.startswith(Configuration.TEMPLATES_DIRECTORY + "/")
Configuration.TEMPLATES_DIRECTORY + '/') pure_name = card_filename[len(Configuration.TEMPLATES_DIRECTORY + "/") :]
pure_name = card_filename[ if pure_name[0] == "s":
len(Configuration.TEMPLATES_DIRECTORY + '/'):]
if pure_name[0] == 's':
return board.SpecialCard(int(pure_name[1])) return board.SpecialCard(int(pure_name[1]))
if pure_name[0] == 'n': if pure_name[0] == "n":
return board.NumberCard( return board.NumberCard(
suit=board.NumberCard.Suit( suit=board.NumberCard.Suit(int(pure_name[1])),
int(pure_name[1])), number=int(pure_name[2])) number=int(pure_name[2]),
)
raise AssertionError() raise AssertionError()
catalogue: List[Tuple[np.ndarray, board.Card]] = [] catalogue: List[Tuple[np.ndarray, board.Card]] = []
with zipfile.ZipFile(filename, 'r') as zip_file: with zipfile.ZipFile(filename, "r") as zip_file:
adj = adjustment.Adjustment( adj = adjustment.Adjustment(
**json.loads( **json.loads(zip_file.read(Configuration.ADJUSTMENT_FILE_NAME))
zip_file.read(Configuration.ADJUSTMENT_FILE_NAME))) )
mydir=tempfile.mkdtemp()
for template_filename in ( for template_filename in (
x for x in zip_file.namelist() if x
x.startswith(Configuration.TEMPLATES_DIRECTORY + '/')): for x in zip_file.namelist()
if x.startswith(Configuration.TEMPLATES_DIRECTORY + "/")
):
myfile = zip_file.extract(template_filename, path=mydir)
catalogue.append( catalogue.append(
(np.load(io.BytesIO(zip_file.read(template_filename))), (
_parse_file_name(template_filename))) cv2.imread(myfile),
_parse_file_name(template_filename),
)
)
assert catalogue[-1][0] is not None assert catalogue[-1][0] is not None
return Configuration(adj=adj, catalogue=catalogue, meta={}) return Configuration(adj=adj, catalogue=catalogue, meta={})
@staticmethod @staticmethod
def generate(image: np.ndarray) -> 'Configuration': def generate(image: np.ndarray) -> "Configuration":
"""Generate a configuration with user input""" """Generate a configuration with user input"""
adj = adjustment.adjust_field(image) adj = adjustment.adjust_field(image)
squares = card_finder.get_field_squares(image, adj, 5, 8) squares = card_finder.get_field_squares(image, adj, 5, 8)

61
test/boards.py
View File

@@ -1,6 +1,8 @@
"""Contains an example board to run tests on""" """Contains an example board to run tests on"""
from shenzhen_solitaire.board import NumberCard, SpecialCard, Board from shenzhen_solitaire.board import NumberCard, SpecialCard, Board
Suit = NumberCard.Suit
TEST_BOARD = Board() TEST_BOARD = Board()
TEST_BOARD.field[0] = [ TEST_BOARD.field[0] = [
SpecialCard.Fa, SpecialCard.Fa,
@@ -65,3 +67,62 @@ TEST_BOARD.field[7] = [
NumberCard(NumberCard.Suit.Black, 1), NumberCard(NumberCard.Suit.Black, 1),
NumberCard(NumberCard.Suit.Green, 8), NumberCard(NumberCard.Suit.Green, 8),
] ]
B20190809172206_1 = Board()
B20190809172206_1.field[0] = [
NumberCard(Suit.Green, 6),
NumberCard(Suit.Green, 5),
NumberCard(Suit.Red, 4),
NumberCard(Suit.Green, 4),
SpecialCard.Fa,
]
B20190809172206_1.field[1] = [
NumberCard(Suit.Black, 8),
NumberCard(Suit.Black, 6),
SpecialCard.Zhong,
NumberCard(Suit.Black, 9),
NumberCard(Suit.Green, 7),
]
B20190809172206_1.field[2] = [
SpecialCard.Zhong,
NumberCard(Suit.Black, 4),
NumberCard(Suit.Green, 2),
SpecialCard.Bai,
SpecialCard.Zhong,
]
B20190809172206_1.field[3] = [
NumberCard(Suit.Green, 1),
NumberCard(Suit.Green, 3),
NumberCard(Suit.Black, 5),
SpecialCard.Fa,
SpecialCard.Fa,
]
B20190809172206_1.field[4] = [
NumberCard(Suit.Red, 8),
SpecialCard.Zhong,
NumberCard(Suit.Red, 7),
]
B20190809172206_1.field[5] = [
SpecialCard.Fa,
SpecialCard.Bai,
NumberCard(Suit.Red, 2),
SpecialCard.Hua,
SpecialCard.Bai,
]
B20190809172206_1.field[6] = [
NumberCard(Suit.Black, 2),
NumberCard(Suit.Green, 8),
NumberCard(Suit.Black, 7),
SpecialCard.Bai,
NumberCard(Suit.Red, 9),
]
B20190809172206_1.field[7] = [
NumberCard(Suit.Red, 3),
NumberCard(Suit.Black, 3),
NumberCard(Suit.Green, 9),
NumberCard(Suit.Red, 5),
NumberCard(Suit.Red, 6),
]

17
test/test_cv.py
View File

@@ -8,17 +8,18 @@ import numpy as np
from shenzhen_solitaire import board from shenzhen_solitaire import board
from shenzhen_solitaire.card_detection import adjustment, board_parser from shenzhen_solitaire.card_detection import adjustment, board_parser
from shenzhen_solitaire.card_detection.configuration import Configuration from shenzhen_solitaire.card_detection.configuration import Configuration
from . import boards
class CardDetectionTest(unittest.TestCase): class CardDetectionTest(unittest.TestCase):
def test_parse(self) -> None: def test_parse(self) -> None:
"""Parse a configuration""" """Parse a configuration and a board"""
with open("pictures/20190809172213_1.jpg", "rb") as png_file: image = cv2.imread("pictures/20190809172206_1.jpg")
img_str = png_file.read()
nparr = np.frombuffer(img_str, np.uint8)
image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
# image = cv2.resize(image, (1000, 629))
loaded_config = Configuration.load("test_config.zip") loaded_config = Configuration.load("test_config.zip")
# loaded_config.field_adjustment = adjustment.adjust_field(image) my_board = board_parser.parse_board(image, loaded_config)
print(board_parser.parse_board(image, loaded_config))
for rows in zip(boards.B20190809172206_1.field, my_board.field):
for good_cell, test_cell in zip(*rows):
self.assertEqual(good_cell, test_cell)

BIN
test_config.zip
View File

Binary file not shown.

8
tools/generate_config.py
View File

@@ -4,11 +4,11 @@ from shenzhen_solitaire.card_detection.configuration import Configuration
def main() -> None: def main() -> None:
"""Generate a configuration""" """Generate a configuration"""
with open("pictures/20190809172213_1.jpg", 'rb') as png_file: image = cv2.imread("pictures/20190809172213_1.jpg")
img_str = png_file.read()
nparr = np.frombuffer(img_str, np.uint8)
image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
generated_config = Configuration.generate(image) generated_config = Configuration.generate(image)
generated_config.save('test_config.zip') generated_config.save('test_config.zip')
if __name__ == "__main__":
main()