python鼠标模拟操作_python模拟鼠标不用坐标

python鼠标模拟操作

通过简单的记录鼠标坐标，并通过计算鼠标下图片的偏移来完成简单的位置校准，用于在微偏移的界面模拟鼠标操作
GitHub链接
https://github.com/WindSnowLi/My-python-tools/tree/main/PseudoOperation

原文：https://www.blog.hiyj.cn/article/detail/76

记录部分

# -*- coding:utf-8 -*-
import datetime
import os
from threading import Thread

import pynput
from PIL import ImageGrab
from pynput.keyboard import Key

# 监听鼠标
last_time = datetime.datetime.now()
operate_record = []
id = 0
status_flag = True


# 截取指定位置
def make_screenshot(x1, y1, x2, y2):
    """截图

    :param x1: 开始截图的x1坐标
    :param y1: 开始截图的x1标
    :param x2: 开始截图的x2坐标
    :param y2: 结束截图的y2坐标
    :return: None
    """
    # id: 图片ID
    global id
    bbox = (x1, y1, x2, y2)
    im = ImageGrab.grab(bbox)
    im.save('./pic/' + str(id) + '.png')  # 保存截图文件的路径


# 监听鼠标移动，并记录移动信息
def on_move(x, y):
    global status_flag
    if not status_flag:
        return status_flag
    global last_time
    global id
    global operate_record
    # 防止读取的鼠标位置越界
    if y > 1080:
        y = 1080
    if x > 1920:
        x = 1920
    if x < 0:
        x = 0
    if y < 0:
        y = 0
    print('Pointer moved to {0}'.format((x, y)))
    id += 1
    # 边缘位置不再进行截图
    if id % 100 == 0 and (x >= 150 or x <= 1920 - 150) and (y > 100 or y < 1080 - 100):
        make_screenshot(x - 75, y - 50, x + 75, y + 50)
    single = {'id': id, 'x': x, 'y': y, "event": "move", "button": "", 'action': '',
              'time': (datetime.datetime.now() - last_time).total_seconds()}
    last_time = datetime.datetime.now()
    operate_record.append(single)


# 监听点击信息
def on_click(x, y, button, pressed):
    global status_flag
    if not status_flag:
        return status_flag
    global last_time
    global id
    global operate_record
    # 监听鼠标点击
    print('{0} at {1}'.format('Pressed' if pressed else 'Released', (x, y)))
    id += 1
    single = {'id': id, 'x': x, 'y': y, "event": "click", "button": str(button),
              'action': 'pressed' if pressed else 'released',
              'time': (datetime.datetime.now() - last_time).total_seconds()}
    last_time = datetime.datetime.now()
    operate_record.append(single)
    make_screenshot(x - 75, y - 50, x + 75, y + 50)


# 监听键盘按键按下
def on_press(key):
    global status_flag
    if not status_flag:
        return status_flag
    # 监听按键
    print('{0} pressed'.format(key))


# 监听按键释放信息
def on_release(key):
    global status_flag
    global operate_record
    # 监听释放
    print('{0} release'.format(key))
    # 若按下的为ESC键，终止程序
    if key == Key.esc:
        with open('./record.txt', 'w') as fp:
            for i in operate_record:
                fp.write(str(i) + '\n')
        print("end")
        status_flag = False
        return status_flag


def listener_mouse():
    with pynput.mouse.Listener(on_move=on_move, on_click=on_click) as listener:
        listener.join()


def listener_key():
    with pynput.keyboard.Listener(on_press=on_press, on_release=on_release)as listener:
        listener.join()


# 一个鼠标监听器是一个线程。线程，所有的回调将从线程调用。从任何地方调用pynput.mouse.Listener.stop，或者调用pynput.mouse.Listener.StopException或从回调中返回False来停止监听器。


if __name__ == '__main__':
    if not os.path.exists('./pic'):
        os.mkdir('./pic')
    t1 = Thread(target=listener_mouse)
    t2 = Thread(target=listener_key)
    t1.start()
    t2.start()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

操作部分

# -*- coding:utf-8 -*-


# 控制鼠标
# 读鼠标坐标
import json
import os
import time
from threading import Thread

import cv2
from PIL import ImageGrab
from pynput.keyboard import Key

import pynput

status_flag = True
mouse = pynput.mouse.Controller()
calibration = [0, 0]


def make_screenshot(x1, y1, x2, y2):
    """截图

    :param x1: 开始截图的x1坐标
    :param y1: 开始截图的x1标
    :param x2: 开始截图的x2坐标
    :param y2: 结束截图的y2坐标
    :return: None
    """
    # id: 图片ID
    global id
    bbox = (x1, y1, x2, y2)
    im = ImageGrab.grab(bbox)
    im.save('./temp/present.png')  # 保存截图文件的路径


def identify_pictures(img1Path, img2Path):
    img = cv2.imread(img1Path, 0)
    img2 = img.copy()
    template = cv2.imread(img2Path, 0)
    w, h = template.shape[::-1]

    meth = 'cv2.TM_CCOEFF'
    img = img2.copy()

    method = eval(meth)

    res = cv2.matchTemplate(img, template, method)
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
    return max_loc


def setPosition(x, y):
    global mouse
    # 本人电脑点击时鼠标坐标会放大偏移，大概会放大1.25倍
    mouse.position = (x / 1.25, y / 1.25)


def setLeftPress():
    global mouse
    mouse.press(pynput.mouse.Button.left)


def setLeftRelease():
    global mouse
    mouse.release(pynput.mouse.Button.left)


def setRightPress():
    global mouse
    mouse.press(pynput.mouse.Button.right)


def setRightRelease():
    global mouse
    mouse.release(pynput.mouse.Button.right)


def on_press(key):
    # 监听按键
    print('{0} pressed'.format(key))


def on_release(key):
    global status_flag
    # 监听释放
    print('{0} release'.format(key))
    if key == Key.esc:
        status_flag = False
        return status_flag


# 通过查找原始鼠标下的图片位置与现在鼠标下的位置进行对比，计算偏移量
def calibration_offset(i):
    make_screenshot(i['x'] - 150 + calibration[0], i['y'] - 100 + calibration[1], i['x'] + 150 + calibration[0],
                    i['y'] + 100 + calibration[1])
    inner_top_left = identify_pictures('./temp/present.png', './pic/' + str(i['id']) + '.png')
    outside = (
        i['x'] - 150 + inner_top_left[0] + calibration[0], i['y'] - 100 + inner_top_left[1] + calibration[1])

    calibration[0] = outside[0] - (i['x'] - 75)
    calibration[1] = outside[1] - (i['y'] - 50)


def operate_mouse():
    global status_flag
    record = []
    with open("./record.txt", 'r') as fp:
        for line in fp:
            record.append(json.loads(line.replace("\'", "\"")))
    for i in record:
        # print(i)
        if not status_flag:
            break
        if i['event'] == 'move':
            if i['id'] % 100 == 0 and (i['x'] >= 200 or i['x'] <= 1920 - 200) and (i['y'] > 100 or i['y'] < 1080 - 100):
                calibration_offset(i)
            i['x'] = i['x'] + calibration[0]
            i['y'] = i['y'] + calibration[1]
            setPosition(i['x'], i['y'])
        elif i['event'] == 'click':
            calibration_offset(i)
            setPosition(i['x'] + calibration[0], i['y'] + calibration[1])
            if i['action'] == 'pressed':
                if i['button'] == 'Button.left':
                    setLeftPress()
                elif i['button'] == 'Button.right':
                    setRightPress()
            elif i['action'] == 'released':
                if i['button'] == 'Button.left':
                    setLeftRelease()
                elif i['button'] == 'Button.right':
                    setRightRelease()
        time.sleep(i['time'])
    status_flag = False
    print("end")


def listener_key():
    with pynput.keyboard.Listener(on_press=on_press, on_release=on_release)as listener:
        listener.join()


if __name__ == '__main__':
    if not os.path.exists('./temp'):
        os.mkdir('./temp')
    t1 = Thread(target=operate_mouse)
    t2 = Thread(target=listener_key)
    t1.start()
    t2.start()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152